[deploy] Merge pull request #52 from microsoft/dev

Dev

DEVELOPMENT.md

@@ -44,7 +44,7 @@ How to set up your local machine.
     ```bash
     yarn start
     ```
-    Open [http://localhost:3000](http://localhost:3000) to view it in the browser.
+    Open [http://localhost:5173](http://localhost:5173) to view it in the browser.
     The page will reload if you make edits. You will also see any lint errors in the console.
 
 ## Build for Production

README.md

@@ -22,6 +22,10 @@ Transform data and create rich visualizations iteratively with AI 🪄. Try Data
 
 ## News 🔥🔥🔥
 
+- [11-07-2024] Minor fun update: data visualization challenges!
+  - We added a few visualization challenges with the sample datasets. Can you complete them all? [[try them out!]](https://github.com/microsoft/data-formulator/issues/53#issue-2641841252)
+  - Comment in the issue when you did, or share your results/questions with others! [[comment here]](https://github.com/microsoft/data-formulator/issues/53)
+
 - [10-11-2024] Data Formulator python package released! 
   - You can now install Data Formulator using Python and run it locally, easily. [[check it out]](#get-started).
   - Our Codespaces configuration is also updated for fast start up ⚡️. [[try it now!]](https://codespaces.new/microsoft/data-formulator?quickstart=1)

py-src/data_formulator/app.py

@@ -5,6 +5,9 @@
 import random
 import sys
 import os
+import mimetypes
+mimetypes.add_type('application/javascript', '.js')
+mimetypes.add_type('application/javascript', '.mjs')
 
 import flask
 from flask import Flask, request, send_from_directory, redirect, url_for
@@ -53,13 +56,45 @@
 @app.route('/vega-datasets')
 def get_example_dataset_list():
     dataset_names = vega_data.list_datasets()
-    example_datasets = ['co2-concentration', 'movies', 'seattle-weather', 
-                        'disasters', 'unemployment-across-industries']
+    example_datasets = [
+        {"name": "gapminder", "challenges": [
+            {"text": "Create a line chart to show the life expectancy trend of each country over time.", "difficulty": "easy"},
+            {"text": "Visualize the top 10 countries with highest life expectancy in 2005.", "difficulty": "medium"},
+            {"text": "Find top 10 countries that have the biggest difference of life expectancy in 1955 and 2005.", "difficulty": "hard"},
+            {"text": "Rank countries by their average population per decade. Then only show countries with population over 50 million in 2005.", "difficulty": "hard"}
+        ]},
+        {"name": "income", "challenges": [
+            {"text": "Create a line chart to show the income trend of each state over time.", "difficulty": "easy"},
+            {"text": "Only show washington and california's percentage of population in each income group each year.", "difficulty": "medium"},
+            {"text": "Find the top 5 states with highest percentage of high income group in 2016.", "difficulty": "hard"}
+        ]},
+        {"name": "disasters", "challenges": [
+            {"text": "Create a scatter plot to show the number of death from each disaster type each year.", "difficulty": "easy"},
+            {"text": "Filter the data and show the number of death caused by flood or drought each year.", "difficulty": "easy"},
+            {"text": "Create a heatmap to show the total number of death caused by each disaster type each decade.", "difficulty": "hard"},
+            {"text": "Exclude 'all natural disasters' from the previous chart.", "difficulty": "medium"}
+        ]},
+        {"name": "movies", "challenges": [
+            {"text": "Create a scatter plot to show the relationship between budget and worldwide gross.", "difficulty": "easy"},
+            {"text": "Find the top 10 movies with highest profit after 2000 and visualize them in a bar chart.", "difficulty": "easy"},
+            {"text": "Visualize the median profit ratio of movies in each genre", "difficulty": "medium"},
+            {"text": "Create a scatter plot to show the relationship between profit and IMDB rating.", "difficulty": "medium"},
+            {"text": "Turn the above plot into a heatmap by bucketing IMDB rating and profit, color tiles by the number of movies in each bucket.", "difficulty": "hard"}
+        ]},
+        {"name": "unemployment-across-industries", "challenges": [
+            {"text": "Create a scatter plot to show the relationship between unemployment rate and year.", "difficulty": "easy"},
+            {"text": "Create a line chart to show the average unemployment per year for each industry.", "difficulty": "medium"},
+            {"text": "Find the 5 most stable industries (least change in unemployment rate between 2000 and 2010) and visualize their trend over time using line charts.", "difficulty": "medium"},
+            {"text": "Create a bar chart to show the unemployment rate change between 2000 and 2010, and highlight the top 5 most stable industries with least change.", "difficulty": "hard"}
+        ]}
+    ]
     dataset_info = []
     print(dataset_names)
-    for name in example_datasets:
+    for dataset in example_datasets:
+        name = dataset["name"]
+        challenges = dataset["challenges"]
         try:
-            info_obj = {'name': name, 'snapshot': vega_data(name).to_json(orient='records')} 
+            info_obj = {'name': name, 'challenges': challenges, 'snapshot': vega_data(name).to_json(orient='records')}
             dataset_info.append(info_obj)
         except:
             pass

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "data_formulator"
-version = "0.1.3.3"
+version = "0.1.4"
 
 requires-python = ">=3.9"
 authors = [

src/app/dfSlice.tsx

@@ -11,6 +11,7 @@ import { getDataTable } from '../views/VisualizationView';
 import { findBaseFields } from '../views/ViewUtils';
 import { adaptChart, getTriggers, getUrls } from './utils';
 import { Type } from '../data/types';
+import { TableChallenges } from '../views/TableSelectionView';
 
 enableMapSet();
 
@@ -34,6 +35,8 @@ export interface DataFormulatorState {
 
     tables : DictTable[];
     charts: Chart[];
+
+    activeChallenges: {tableId: string, challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]}[];
 
     conceptShelfItems: FieldItem[];
 
@@ -66,6 +69,8 @@ const initialState: DataFormulatorState = {
 
     tables: [],
     charts: [],
+
+    activeChallenges: [],
 
     conceptShelfItems: [],
 
@@ -222,6 +227,7 @@ export const dataFormulatorSlice = createSlice({
 
             state.tables = [];
             state.charts = [];
+            state.activeChallenges = [];
 
             state.conceptShelfItems = [];
 
@@ -248,6 +254,8 @@ export const dataFormulatorSlice = createSlice({
             //state.table = undefined;
             state.tables = savedState.tables || [];
             state.charts = savedState.charts || [];
+
+            state.activeChallenges = savedState.activeChallenges || [];
 
             state.conceptShelfItems = savedState.conceptShelfItems || [];
 
@@ -306,6 +314,9 @@ export const dataFormulatorSlice = createSlice({
             // separate this, so that we only delete on tier of table a time
             state.charts = state.charts.filter(c => !(c.intermediate && c.intermediate.resultTableId == tableId));
         },
+        addChallenges: (state, action: PayloadAction<{tableId: string, challenges: { text: string; difficulty: 'easy' | 'medium' | 'hard'; }[]}>) => {
+            state.activeChallenges = [...state.activeChallenges, action.payload];
+        },
         createNewChart: (state, action: PayloadAction<{chartType?: string, tableId?: string}>) => {
             let chartType = action.payload.chartType;
             let tableId = action.payload.tableId || state.tables[0].id;

src/components/ChartTemplates.tsx

@@ -55,7 +55,7 @@ const tablePlots: ChartTemplate[] = [
         "chart": "Table",
         "icon": chartIconTable,
         "template": { },
-        "channels": ["field 1", "field 2", "field 3", "field 4", "field 5", 'field 6'],
+        "channels": [], //"field 1", "field 2", "field 3", "field 4", "field 5", 'field 6'
         "paths": { }
     },
 ]

src/views/EncodingShelfCard.tsx

@@ -433,7 +433,10 @@ export const EncodingShelfCard: FC<EncodingShelfCardProps> = function ({ chartId
                                     newChart.intermediate = undefined;
                                 }
 
-                                newChart = resolveChartFields(newChart, currentConcepts, refinedGoal, candidateTable);
+                                // there is no need to resolve fields for table chart, just display all fields
+                                if (chart.chartType != "Table") {   
+                                    newChart = resolveChartFields(newChart, currentConcepts, refinedGoal, candidateTable);
+                                }
 
                                 dispatch(dfActions.addChart(newChart));
                                 dispatch(dfActions.setFocusedChart(newChart.id));                                

src/views/MessageSnackbar.tsx

@@ -8,9 +8,9 @@ import IconButton from '@mui/material/IconButton';
 import CloseIcon from '@mui/icons-material/Close';
 import { DataFormulatorState, dfActions } from '../app/dfSlice';
 import { useDispatch, useSelector } from 'react-redux';
-import { Alert, Box, Tooltip, Typography } from '@mui/material';
+import { Alert, alpha, Box, Paper, Tooltip, Typography } from '@mui/material';
 import InfoIcon from '@mui/icons-material/Info';
-
+import AssignmentIcon from '@mui/icons-material/Assignment';
 
 export interface Message {
     type: "success" | "info" | "error",
@@ -22,11 +22,14 @@ export interface Message {
 
 export function MessageSnackbar() {
 
+    const challenges = useSelector((state: DataFormulatorState) => state.activeChallenges);
     const messages = useSelector((state: DataFormulatorState) => state.messages);
     const displayedMessageIdx = useSelector((state: DataFormulatorState) => state.displayedMessageIdx);
     const dispatch = useDispatch();
+    const tables = useSelector((state: DataFormulatorState) => state.tables);
 
     const [open, setOpen] = React.useState(false);
+    const [openChallenge, setOpenChallenge] = React.useState(true);
     const [message, setMessage] = React.useState<Message | undefined>();
 
     React.useEffect(()=>{
@@ -60,13 +63,97 @@ export function MessageSnackbar() {
     let timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone;
     let timestamp = message == undefined ? "" : new Date((message as Message).timestamp).toLocaleString('en-US', { timeZone, hour: "2-digit", minute: "2-digit", second: "2-digit" });
 
+    console.log(challenges);
+    let challenge = challenges.find(c => tables.find(t => t.id == c.tableId));
+
     return (
         <Box>
-            <Tooltip placement="right" title="view last message"><IconButton disabled={messages.length == 0} sx={{position: "absolute", bottom: 16, right: 0}}
-                onClick={()=>{
-                    setOpen(true);
-                    setMessage(messages[messages.length - 1]);
-            }}><InfoIcon /></IconButton></Tooltip>
+            <Tooltip placement="right" title="view challenges">
+                <IconButton 
+                    color="warning"
+                    disabled={challenges.length === 0}
+                    sx={{
+                        position: "absolute", 
+                        bottom: 56, 
+                        right: 8,
+                        animation: challenges.length > 0 ? 'glow 1.5s ease-in-out infinite alternate' : 'none',
+                        '@keyframes glow': {
+                            from: {
+                                boxShadow: '0 0 5px #fff, 0 0 10px #fff, 0 0 15px #ed6c02'
+                            },
+                            to: {
+                                boxShadow: '0 0 10px #fff, 0 0 20px #fff, 0 0 30px #ed6c02'
+                            }
+                        }
+                    }}
+                    onClick={() => setOpenChallenge(true)}
+                >
+                    <AssignmentIcon />
+                </IconButton>
+            </Tooltip>
+            <Tooltip placement="right" title="view last message">
+                <IconButton disabled={messages.length == 0} sx={{position: "absolute", bottom: 16, right: 8}}
+                    onClick={()=>{
+                        setOpen(true);
+                        setMessage(messages[messages.length - 1]);
+                    }}
+                >
+                    <InfoIcon />
+                </IconButton>
+            </Tooltip>
+            {challenge != undefined ? <Snackbar
+                open={openChallenge}
+                anchorOrigin={{vertical: 'bottom', horizontal: 'right'}}
+                sx={{maxWidth: '400px'}}
+            >
+                <Paper sx={{
+                    width: '100%',
+                    bgcolor: 'white',
+                    color: 'text.primary',
+                    p: 2,
+                    boxShadow: 2,
+                    borderRadius: 1,
+                    border: '1px solid #e0e0e0',
+                    display: 'flex',
+                    flexDirection: 'column'
+                }}>
+                    <Box sx={{display: 'flex', justifyContent: 'space-between', alignItems: 'center', mb: 1}}>
+                        <Typography variant="subtitle1" sx={{fontWeight: 'bold', fontSize: 14}}>
+                            Visualization challenges for dataset <Box component="span" sx={{fontWeight: 'bold', color: 'primary.main'}}>{challenge.tableId}</Box>
+                        </Typography>
+                        <IconButton
+                            size="small"
+                            aria-label="close"
+                            onClick={() => setOpenChallenge(false)}
+                        >
+                            <CloseIcon fontSize="small" />
+                        </IconButton>
+                    </Box>
+                    <Box sx={{mb: 2}}>
+                        {challenge.challenges.map((ch, j) => (
+                            <Typography 
+                                key={j} 
+                                variant="body2" 
+                                sx={{
+                                    fontSize: 12,
+                                    marginBottom: 1,
+                                    color: ch.difficulty === 'easy' ? 'success.main'
+                                        : ch.difficulty === 'medium' ? 'warning.main' 
+                                        : 'error.main'
+                                }}
+                            >
+                                <Box 
+                                    component="span" 
+                                    sx={{fontWeight: 'bold'}}
+                                >
+                                    [{ch.difficulty}]
+                                </Box>
+                                {' '}{ch.text}
+                            </Typography>
+                        ))}
+                    </Box>
+                </Paper>
+            </Snackbar> : ""}
             {message != undefined ? <Snackbar
                 open={open && message != undefined}
                 autoHideDuration={message?.type == "error" ? 15000 : 5000}

src/views/SelectableDataGrid.tsx

@@ -29,6 +29,7 @@ import { FieldSource } from '../components/ComponentType';
 
 import ChevronLeftIcon from '@mui/icons-material/ChevronLeft';
 import DeleteIcon from '@mui/icons-material/Delete';
+import FileDownloadIcon from '@mui/icons-material/FileDownload';
 import { dfActions, dfSelectors } from '../app/dfSlice';
 import { useDispatch, useSelector } from 'react-redux';
 import { getUrls } from '../app/utils';
@@ -266,13 +267,14 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ rows, ta
                     {`${rowsToDisplay.length} matches`}   
                 </Typography>: ''}
             </Box>
-            <Tooltip key="delete-action" title={`Delete ${tableName}\n(note: all charts and concepts based on this table will be deleted)`}>
+            {/* <Tooltip key="delete-action" title={`Delete ${tableName}\n(note: all charts and concepts based on this table will be deleted)`}>
                 <IconButton size="small" color="warning" sx={{marginRight: 1}} onClick={() => {
                     dispatch(dfActions.deleteTable(tableName))
                 }}>
                     <DeleteIcon/>
                 </IconButton>
-            </Tooltip>
+            </Tooltip> */}
+
             <IconButton size="small" color="primary"
                 onClick={() => {
                         console.log(`[fyi] just sent request to process load data`);
@@ -452,6 +454,35 @@ export const SelectableDataGrid: React.FC<SelectableDataGridProps> = ({ rows, ta
                     {footerActionsItems}
                 </Collapse>
                 <Box sx={{display: 'flex', alignItems: 'center',  marginRight: 1}}>
+                    <Tooltip title={`Download ${tableName} as CSV`}>
+                        <IconButton size="small" color="primary" sx={{marginRight: 1}}
+                            onClick={() => {
+                                // Create CSV content
+                                const csvContent = [
+                                    Object.keys(rows[0]).join(','), // Header row
+                                    ...rows.map(row => Object.values(row).map(value => 
+                                        // Handle values that need quotes (contain commas or quotes)
+                                        typeof value === 'string' && (value.includes(',') || value.includes('"')) 
+                                            ? `"${value.replace(/"/g, '""')}"` 
+                                            : value
+                                    ).join(','))
+                                ].join('\n');
+
+                                // Create and trigger download
+                                const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' });
+                                const link = document.createElement('a');
+                                const url = URL.createObjectURL(blob);
+                                link.setAttribute('href', url);
+                                link.setAttribute('download', `${tableName}.csv`);
+                                link.style.visibility = 'hidden';
+                                document.body.appendChild(link);
+                                link.click();
+                                document.body.removeChild(link);
+                            }}
+                        >
+                            <FileDownloadIcon/>
+                        </IconButton>
+                    </Tooltip>
                     <Typography  className="table-footer-number">
                         {`${rows.length} rows`}
                     </Typography>