diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 80a6dfd..2b24bde 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -44,7 +44,7 @@ How to set up your local machine. ```bash yarn start ``` - Open [http://localhost:3000](http://localhost:3000) to view it in the browser. + Open [http://localhost:5173](http://localhost:5173) to view it in the browser. The page will reload if you make edits. You will also see any lint errors in the console. ## Build for Production diff --git a/README.md b/README.md index a919411..116e3e1 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ Transform data and create rich visualizations iteratively with AI 🪄. Try Data ## News 🔥🔥🔥 +- [11-07-2024] Minor fun update: data visualization challenges! + - We added a few visualization challenges with the sample datasets. Can you complete them all? [[try them out!]](https://github.com/microsoft/data-formulator/issues/53#issue-2641841252) + - Comment in the issue when you did, or share your results/questions with others! [[comment here]](https://github.com/microsoft/data-formulator/issues/53) + - [10-11-2024] Data Formulator python package released! - You can now install Data Formulator using Python and run it locally, easily. [[check it out]](#get-started). - Our Codespaces configuration is also updated for fast start up ⚡️. [[try it now!]](https://codespaces.new/microsoft/data-formulator?quickstart=1) diff --git a/py-src/data_formulator/app.py b/py-src/data_formulator/app.py index 032edef..1c89baf 100644 --- a/py-src/data_formulator/app.py +++ b/py-src/data_formulator/app.py @@ -5,6 +5,9 @@ import random import sys import os +import mimetypes +mimetypes.add_type('application/javascript', '.js') +mimetypes.add_type('application/javascript', '.mjs') import flask from flask import Flask, request, send_from_directory, redirect, url_for @@ -53,13 +56,45 @@ @app.route('/vega-datasets') def get_example_dataset_list(): dataset_names = vega_data.list_datasets() - example_datasets = ['co2-concentration', 'movies', 'seattle-weather', - 'disasters', 'unemployment-across-industries'] + example_datasets = [ + {"name": "gapminder", "challenges": [ + {"text": "Create a line chart to show the life expectancy trend of each country over time.", "difficulty": "easy"}, + {"text": "Visualize the top 10 countries with highest life expectancy in 2005.", "difficulty": "medium"}, + {"text": "Find top 10 countries that have the biggest difference of life expectancy in 1955 and 2005.", "difficulty": "hard"}, + {"text": "Rank countries by their average population per decade. Then only show countries with population over 50 million in 2005.", "difficulty": "hard"} + ]}, + {"name": "income", "challenges": [ + {"text": "Create a line chart to show the income trend of each state over time.", "difficulty": "easy"}, + {"text": "Only show washington and california's percentage of population in each income group each year.", "difficulty": "medium"}, + {"text": "Find the top 5 states with highest percentage of high income group in 2016.", "difficulty": "hard"} + ]}, + {"name": "disasters", "challenges": [ + {"text": "Create a scatter plot to show the number of death from each disaster type each year.", "difficulty": "easy"}, + {"text": "Filter the data and show the number of death caused by flood or drought each year.", "difficulty": "easy"}, + {"text": "Create a heatmap to show the total number of death caused by each disaster type each decade.", "difficulty": "hard"}, + {"text": "Exclude 'all natural disasters' from the previous chart.", "difficulty": "medium"} + ]}, + {"name": "movies", "challenges": [ + {"text": "Create a scatter plot to show the relationship between budget and worldwide gross.", "difficulty": "easy"}, + {"text": "Find the top 10 movies with highest profit after 2000 and visualize them in a bar chart.", "difficulty": "easy"}, + {"text": "Visualize the median profit ratio of movies in each genre", "difficulty": "medium"}, + {"text": "Create a scatter plot to show the relationship between profit and IMDB rating.", "difficulty": "medium"}, + {"text": "Turn the above plot into a heatmap by bucketing IMDB rating and profit, color tiles by the number of movies in each bucket.", "difficulty": "hard"} + ]}, + {"name": "unemployment-across-industries", "challenges": [ + {"text": "Create a scatter plot to show the relationship between unemployment rate and year.", "difficulty": "easy"}, + {"text": "Create a line chart to show the average unemployment per year for each industry.", "difficulty": "medium"}, + {"text": "Find the 5 most stable industries (least change in unemployment rate between 2000 and 2010) and visualize their trend over time using line charts.", "difficulty": "medium"}, + {"text": "Create a bar chart to show the unemployment rate change between 2000 and 2010, and highlight the top 5 most stable industries with least change.", "difficulty": "hard"} + ]} + ] dataset_info = [] print(dataset_names) - for name in example_datasets: + for dataset in example_datasets: + name = dataset["name"] + challenges = dataset["challenges"] try: - info_obj = {'name': name, 'snapshot': vega_data(name).to_json(orient='records')} + info_obj = {'name': name, 'challenges': challenges, 'snapshot': vega_data(name).to_json(orient='records')} dataset_info.append(info_obj) except: pass diff --git a/pyproject.toml b/pyproject.toml index 659e89a..b677c99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "data_formulator" -version = "0.1.3.3" +version = "0.1.4" requires-python = ">=3.9" authors = [ diff --git a/src/app/dfSlice.tsx b/src/app/dfSlice.tsx index a148597..b306745 100644 --- a/src/app/dfSlice.tsx +++ b/src/app/dfSlice.tsx @@ -11,6 +11,7 @@ import { getDataTable } from '../views/VisualizationView'; import { findBaseFields } from '../views/ViewUtils'; import { adaptChart, getTriggers, getUrls } from './utils'; import { Type } from '../data/types'; +import { TableChallenges } from '../views/TableSelectionView'; enableMapSet(); @@ -34,6 +35,8 @@ export interface DataFormulatorState { tables : DictTable[]; charts: Chart[]; + + activeChallenges: {tableId: string, challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]}[]; conceptShelfItems: FieldItem[]; @@ -66,6 +69,8 @@ const initialState: DataFormulatorState = { tables: [], charts: [], + + activeChallenges: [], conceptShelfItems: [], @@ -222,6 +227,7 @@ export const dataFormulatorSlice = createSlice({ state.tables = []; state.charts = []; + state.activeChallenges = []; state.conceptShelfItems = []; @@ -248,6 +254,8 @@ export const dataFormulatorSlice = createSlice({ //state.table = undefined; state.tables = savedState.tables || []; state.charts = savedState.charts || []; + + state.activeChallenges = savedState.activeChallenges || []; state.conceptShelfItems = savedState.conceptShelfItems || []; @@ -306,6 +314,9 @@ export const dataFormulatorSlice = createSlice({ // separate this, so that we only delete on tier of table a time state.charts = state.charts.filter(c => !(c.intermediate && c.intermediate.resultTableId == tableId)); }, + addChallenges: (state, action: PayloadAction<{tableId: string, challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]}>) => { + state.activeChallenges = [...state.activeChallenges, action.payload]; + }, createNewChart: (state, action: PayloadAction<{chartType?: string, tableId?: string}>) => { let chartType = action.payload.chartType; let tableId = action.payload.tableId || state.tables[0].id; diff --git a/src/components/ChartTemplates.tsx b/src/components/ChartTemplates.tsx index a3b16d8..b54970c 100644 --- a/src/components/ChartTemplates.tsx +++ b/src/components/ChartTemplates.tsx @@ -55,7 +55,7 @@ const tablePlots: ChartTemplate[] = [ "chart": "Table", "icon": chartIconTable, "template": { }, - "channels": ["field 1", "field 2", "field 3", "field 4", "field 5", 'field 6'], + "channels": [], //"field 1", "field 2", "field 3", "field 4", "field 5", 'field 6' "paths": { } }, ] diff --git a/src/views/EncodingShelfCard.tsx b/src/views/EncodingShelfCard.tsx index c35e87e..935a19e 100644 --- a/src/views/EncodingShelfCard.tsx +++ b/src/views/EncodingShelfCard.tsx @@ -433,7 +433,10 @@ export const EncodingShelfCard: FC = function ({ chartId newChart.intermediate = undefined; } - newChart = resolveChartFields(newChart, currentConcepts, refinedGoal, candidateTable); + // there is no need to resolve fields for table chart, just display all fields + if (chart.chartType != "Table") { + newChart = resolveChartFields(newChart, currentConcepts, refinedGoal, candidateTable); + } dispatch(dfActions.addChart(newChart)); dispatch(dfActions.setFocusedChart(newChart.id)); diff --git a/src/views/MessageSnackbar.tsx b/src/views/MessageSnackbar.tsx index 433c5fe..66ae8a3 100644 --- a/src/views/MessageSnackbar.tsx +++ b/src/views/MessageSnackbar.tsx @@ -8,9 +8,9 @@ import IconButton from '@mui/material/IconButton'; import CloseIcon from '@mui/icons-material/Close'; import { DataFormulatorState, dfActions } from '../app/dfSlice'; import { useDispatch, useSelector } from 'react-redux'; -import { Alert, Box, Tooltip, Typography } from '@mui/material'; +import { Alert, alpha, Box, Paper, Tooltip, Typography } from '@mui/material'; import InfoIcon from '@mui/icons-material/Info'; - +import AssignmentIcon from '@mui/icons-material/Assignment'; export interface Message { type: "success" | "info" | "error", @@ -22,11 +22,14 @@ export interface Message { export function MessageSnackbar() { + const challenges = useSelector((state: DataFormulatorState) => state.activeChallenges); const messages = useSelector((state: DataFormulatorState) => state.messages); const displayedMessageIdx = useSelector((state: DataFormulatorState) => state.displayedMessageIdx); const dispatch = useDispatch(); + const tables = useSelector((state: DataFormulatorState) => state.tables); const [open, setOpen] = React.useState(false); + const [openChallenge, setOpenChallenge] = React.useState(true); const [message, setMessage] = React.useState(); React.useEffect(()=>{ @@ -60,13 +63,97 @@ export function MessageSnackbar() { let timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone; let timestamp = message == undefined ? "" : new Date((message as Message).timestamp).toLocaleString('en-US', { timeZone, hour: "2-digit", minute: "2-digit", second: "2-digit" }); + console.log(challenges); + let challenge = challenges.find(c => tables.find(t => t.id == c.tableId)); + return ( - { - setOpen(true); - setMessage(messages[messages.length - 1]); - }}> + + 0 ? 'glow 1.5s ease-in-out infinite alternate' : 'none', + '@keyframes glow': { + from: { + boxShadow: '0 0 5px #fff, 0 0 10px #fff, 0 0 15px #ed6c02' + }, + to: { + boxShadow: '0 0 10px #fff, 0 0 20px #fff, 0 0 30px #ed6c02' + } + } + }} + onClick={() => setOpenChallenge(true)} + > + + + + + { + setOpen(true); + setMessage(messages[messages.length - 1]); + }} + > + + + + {challenge != undefined ? + + + + Visualization challenges for dataset {challenge.tableId} + + setOpenChallenge(false)} + > + + + + + {challenge.challenges.map((ch, j) => ( + + + [{ch.difficulty}] + + {' '}{ch.text} + + ))} + + + : ""} {message != undefined ? = ({ rows, ta {`${rowsToDisplay.length} matches`} : ''} - + {/* { dispatch(dfActions.deleteTable(tableName)) }}> - + */} + { console.log(`[fyi] just sent request to process load data`); @@ -452,6 +454,35 @@ export const SelectableDataGrid: React.FC = ({ rows, ta {footerActionsItems} + + { + // Create CSV content + const csvContent = [ + Object.keys(rows[0]).join(','), // Header row + ...rows.map(row => Object.values(row).map(value => + // Handle values that need quotes (contain commas or quotes) + typeof value === 'string' && (value.includes(',') || value.includes('"')) + ? `"${value.replace(/"/g, '""')}"` + : value + ).join(',')) + ].join('\n'); + + // Create and trigger download + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); + const link = document.createElement('a'); + const url = URL.createObjectURL(blob); + link.setAttribute('href', url); + link.setAttribute('download', `${tableName}.csv`); + link.style.visibility = 'hidden'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }} + > + + + {`${rows.length} rows`} diff --git a/src/views/TableSelectionView.tsx b/src/views/TableSelectionView.tsx index fb84a0f..da2d5d1 100644 --- a/src/views/TableSelectionView.tsx +++ b/src/views/TableSelectionView.tsx @@ -68,15 +68,23 @@ function a11yProps(index: number) { }; } + +export interface TableChallenges { + name: string; + challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]; + table: DictTable; +} + + export interface TableSelectionViewProps { - tables: DictTable[]; + tableChallenges: TableChallenges[]; handleDeleteTable?: (index: number) => void; - handleSelectTable: (table: DictTable) => void; + handleSelectTable: (tableChallenges: TableChallenges) => void; hideRowNum?: boolean; } -export const TableSelectionView: React.FC = function TableSelectionView({ tables, handleDeleteTable, handleSelectTable, hideRowNum }) { +export const TableSelectionView: React.FC = function TableSelectionView({ tableChallenges, handleDeleteTable, handleSelectTable, hideRowNum }) { const [value, setValue] = React.useState(0); @@ -85,9 +93,9 @@ export const TableSelectionView: React.FC = function Ta }; let tabTitiles : string[] = []; - for (let i = 0; i < tables.length; i ++) { + for (let i = 0; i < tableChallenges.length; i ++) { let k = 0; - let title = tables[i].id; + let title = tableChallenges[i].name; while (tabTitiles.includes(title)) { k = k + 1; title = `${title}_${k}`; @@ -96,7 +104,7 @@ export const TableSelectionView: React.FC = function Ta } return ( - + = function Ta > {tabTitiles.map((title, i) => )} - {tables.map((t, i) => { + {tableChallenges.map((tc, i) => { + let t = tc.table; let sampleRows = [...t.rows.slice(0,9), Object.fromEntries(t.names.map(n => [n, "..."]))]; let colDefs = t.names.map(name => { return { id: name, label: name, minWidth: 60, align: undefined, format: (v: any) => v, }}) + + let challengeView = + Try these data visualization challenges with this dataset: + {tc.challenges.map((c, j) => + [{c.difficulty}] {c.text} + )} + + let content = + {challengeView} + return {content} @@ -131,7 +152,7 @@ export const TableSelectionView: React.FC = function Ta } @@ -145,7 +166,7 @@ export const TableSelectionView: React.FC = function Ta export const TableSelectionDialog: React.FC<{ buttonElement: any }> = function TableSelectionDialog({ buttonElement }) { - const [datasetPreviews, setDatasetPreviews] = React.useState([]); + const [datasetPreviews, setDatasetPreviews] = React.useState([]); const [tableDialogOpen, setTableDialogOpen] = useState(false); React.useEffect(() => { @@ -153,11 +174,11 @@ export const TableSelectionDialog: React.FC<{ buttonElement: any }> = function T fetch(`${getUrls().VEGA_DATASET_LIST}`) .then((response) => response.json()) .then((result) => { - let tables : DictTable[] = result.map((info: any) => { + let tableChallenges : TableChallenges[] = result.map((info: any) => { let table = createTableFromFromObjectArray(info["name"], JSON.parse(info["snapshot"])) - return table - }).filter((t : DictTable | undefined) => t != undefined); - setDatasetPreviews(tables); + return {table: table, challenges: info["challenges"], name: info["name"]} + }).filter((t : TableChallenges | undefined) => t != undefined); + setDatasetPreviews(tableChallenges); }); // No variable dependencies means this would run only once after the first render }, []); @@ -187,21 +208,25 @@ export const TableSelectionDialog: React.FC<{ buttonElement: any }> = function T - { + handleSelectTable={(tableChallenges) => { // request public datasets from the server - console.log(tableInfo); - console.log(`${getUrls().VEGA_DATASET_REQUEST_PREFIX}${tableInfo.id}`) - fetch(`${getUrls().VEGA_DATASET_REQUEST_PREFIX}${tableInfo.id}`) + console.log(tableChallenges); + console.log(`${getUrls().VEGA_DATASET_REQUEST_PREFIX}${tableChallenges.table.id}`) + fetch(`${getUrls().VEGA_DATASET_REQUEST_PREFIX}${tableChallenges.table.id}`) .then((response) => { return response.text() }) .then((text) => { - let fullTable = createTableFromFromObjectArray(tableInfo.id, JSON.parse(text)); + let fullTable = createTableFromFromObjectArray(tableChallenges.table.id, JSON.parse(text)); if (fullTable) { dispatch(dfActions.addTable(fullTable)); dispatch(fetchFieldSemanticType(fullTable)); + dispatch(dfActions.addChallenges({ + tableId: tableChallenges.table.id, + challenges: tableChallenges.challenges + })); } else { throw ""; } @@ -212,7 +237,7 @@ export const TableSelectionDialog: React.FC<{ buttonElement: any }> = function T dispatch(dfActions.addMessages({ "timestamp": Date.now(), "type": "error", - "value": `Unable to load the sample dataset ${tableInfo.id}, please try again later or upload your data.` + "value": `Unable to load the sample dataset ${tableChallenges.table.id}, please try again later or upload your data.` })); }) }}/>