Skip to content

Commit

Permalink
🔨 crudely working version as overlay on CTRL+N
Browse files Browse the repository at this point in the history
  • Loading branch information
@danyx23
danyx23 committed Aug 12, 2024
1 parent 941f726 commit 4a1192b
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 266 deletions.
53 changes: 52 additions & 1 deletion packages/@ourworldindata/grapher/src/core/Grapher.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ import {
type EntitySelectorState,
} from "../entitySelector/EntitySelector"
import { SlideInDrawer } from "../slideInDrawer/SlideInDrawer"

declare const StarboardEmbed: any // Inform TypeScript about StarboardEmbed
declare global {
interface Window {
details?: DetailDictionary
Expand Down Expand Up @@ -2261,6 +2261,49 @@ export class Grapher
void this.timelineController.togglePlay()
}

@action.bound private async addNotebook() {
const notebookNode = document.createElement("div")
notebookNode.id = "notebook"
document.body.appendChild(notebookNode)
const packageUrl = "https://unpkg.com/starboard-wrap/dist/index.js"

const module = await import(packageUrl)

// Ensure the module is available and extract the necessary export
const { StarboardEmbed } = module

// Create a script tag to load the necessary library
const notebookContent = `# %% [markdown]
# Setup code
The code in the cell below provides the infrastructure to access our data. It is automatically run when the page is loaded.
# %%--- [python]
# properties:
# run_on_load: true
# ---%%
from pyodide.http import open_url
from io import StringIO
csv = open_url("${this.baseUrl}.csv").getValue()
df = pd.read_csv(StringIO(csv))
# %% [markdown]
Add your code below - the example shows the first few rows of the data. Refer to the pandas documentation of ask ChatGPT for help on how to manipulate pandas dataframes.
# %%--- [python]
# properties:
# run_on_load: true
# ---%%
df.head()
`

const mount = document.querySelector("#notebook")
const el = new StarboardEmbed({
notebookContent: notebookContent,
src: "https://unpkg.com/starboard-notebook/dist/index.html",
})

mount!.appendChild(el)
}

selection =
this.manager?.selection ??
new SelectionArray(
Expand Down Expand Up @@ -2362,6 +2405,14 @@ export class Grapher
title: "Reset to original",
category: "Navigation",
},
{
combo: "ctrl+n",
fn: (): void => {
void this.addNotebook()
},
title: "Open a notebook",
category: "Notebook",
},
]

if (this.slideShow) {
Expand Down
11 changes: 11 additions & 0 deletions packages/@ourworldindata/grapher/src/core/grapher.scss
View file
Original file line number Diff line number Diff line change
Expand Up @@ -247,3 +247,14 @@ $zindex-controls-drawer: 150;
@include dod-span;
}
}

#notebook {
position: absolute;
top: 70px;
left: 50px;
right: 50px;
bottom: 50px;
background: white;
z-index: 100;
overflow-y: scroll;
}
8 changes: 0 additions & 8 deletions site/DataPageV2.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import { SiteFooter } from "./SiteFooter.js"
import { SiteHeader } from "./SiteHeader.js"
import { IFrameDetector } from "./IframeDetector.js"
import { DebugProvider } from "./gdocs/DebugContext.js"
import { getNotebookScript } from "./GrapherPage.js"

export const DataPageV2 = (props: {
grapher: GrapherInterface | undefined
Expand Down Expand Up @@ -174,7 +173,6 @@ export const DataPageV2 = (props: {
</DebugProvider>
</div>
</main>
<div id="notebook" />
<SiteFooter
baseUrl={baseUrl}
context={SiteFooterContext.dataPageV2}
Expand All @@ -187,12 +185,6 @@ export const DataPageV2 = (props: {
)}`,
}}
/>
<script
type="module"
dangerouslySetInnerHTML={{
__html: getNotebookScript(grapherConfig.slug!),
}}
/>
</body>
</html>
)
Expand Down
257 changes: 0 additions & 257 deletions site/GrapherPage.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,256 +29,6 @@ import { SiteFooter } from "./SiteFooter.js"
import { SiteHeader } from "./SiteHeader.js"
import GrapherImage from "./GrapherImage.js"

const notebook = `
import datetime as dt
import json
import re
from pyodide.http import open_url
from dataclasses import dataclass
from typing import Dict, List, Literal, Optional
import pandas as pd
from dateutil.parser import parse as date_parse
class LicenseError(Exception):
pass
class ChartNotFoundError(Exception):
pass
@dataclass
class _Indicator:
data: dict
metadata: dict
def to_dict(self):
return {"data": self.data, "metadata": self.metadata}
def to_frame(self):
if self.metadata.get("nonRedistributable"):
raise LicenseError(
"API download is disallowed for this indicator due to license restrictions from the data provider"
)
# getting a data frame is easy
df = pd.DataFrame.from_dict(self.data)
# turning entity ids into entity names
entities = pd.DataFrame.from_records(self.metadata["dimensions"]["entities"]["values"])
id_to_name = entities.set_index("id").name.to_dict()
df["entities"] = df.entities.apply(id_to_name.__getitem__)
# make the "values" column more interestingly named
short_name = self.metadata.get("shortName", f'_{self.metadata["id"]}')
df = df.rename(columns={"values": short_name})
time_col = self._detect_time_col_type()
if time_col == "dates":
df["years"] = self._convert_years_to_dates(df["years"])
# order the columns better
cols = ["entities", "years"] + sorted(df.columns.difference(["entities", "years"]))
df = df[cols]
return df
def _detect_time_col_type(self) -> Literal["dates", "years"]:
if self.metadata.get("display", {}).get("yearIsDay"):
return "dates"
return "years"
def _convert_years_to_dates(self, years):
base_date = date_parse(self.metadata["display"]["zeroDay"])
return years.apply(lambda y: base_date + dt.timedelta(days=y))
@dataclass
class _GrapherBundle:
config: dict
dimensions: Dict[int, _Indicator]
origins: List[dict]
def to_json(self):
return json.dumps(
{
"config": self.config,
"dimensions": {k: i.to_dict() for k, i in self.dimensions.items()},
"origins": self.origins,
}
)
def size(self):
return len(self.to_json())
@property
def indicators(self) -> List[_Indicator]:
return list(self.dimensions.values())
def to_frame(self):
# combine all the indicators into a single data frame and one metadata dict
metadata = {}
df = None
for i in self.indicators:
to_merge = i.to_frame()
(value_col,) = to_merge.columns.difference(["entities", "years"])
metadata[value_col] = i.metadata.copy()
if df is None:
df = to_merge
else:
df = pd.merge(df, to_merge, how="outer", on=["entities", "years"])
assert df is not None
# save some useful metadata onto the frame
assert self.config
slug = self.config["slug"]
df.attrs["slug"] = slug
df.attrs["url"] = f"https://ourworldindata.org/grapher/{slug}"
df.attrs["metadata"] = metadata
# if there is only one indicator, we can use the slug as the column name
if len(df.columns) == 3:
assert self.config
(value_col,) = df.columns.difference(["entities", "years"])
short_name = slug.replace("-", "_")
df = df.rename(columns={value_col: short_name})
df.attrs["metadata"][short_name] = df.attrs["metadata"].pop(value_col)
df.attrs["value_col"] = short_name
# we kept using "years" until now to keep the code paths the same, but they could
# be dates
if df["years"].astype(str).str.match(r"^\d{4}-\d{2}-\d{2}$").all():
df = df.rename(columns={"years": "dates"})
return df
def __repr__(self):
return f"GrapherBundle(config={self.config}, dimensions=..., origins=...)"
def fetch_json(url):
resp = open_url(url).getvalue()
return json.loads(resp)
def _fetch_grapher_config(slug):
response = open_url(f"https://ourworldindata.org/grapher/{slug}").getvalue()
return json.loads(response.split("//EMBEDDED_JSON")[1])
def _fetch_dimension(id: int) -> _Indicator:
data = fetch_json(f"https://api.ourworldindata.org/v1/indicators/{id}.data.json")
metadata = fetch_json(f"https://api.ourworldindata.org/v1/indicators/{id}.metadata.json")
return _Indicator(data, metadata)
def _fetch_bundle(slug: str) -> _GrapherBundle:
config = _fetch_grapher_config(slug)
indicator_ids = [d["variableId"] for d in config["dimensions"]]
dimensions = {indicator_id: _fetch_dimension(indicator_id) for indicator_id in indicator_ids}
origins = []
for d in dimensions.values():
if d.metadata.get("origins"):
origins.append(d.metadata.pop("origins"))
return _GrapherBundle(config, dimensions, origins)
def _list_charts() -> List[str]:
content = open_url("https://ourworldindata.org/charts").getvalue()
links = re.findall('"(/grapher/[^"]+)"', content)
slugs = [link.strip('"').split("/")[-1] for link in links]
return sorted(set(slugs))
@dataclass
class Chart:
"""
A chart published on Our World in Data, for example:
https://ourworldindata.org/grapher/life-expectancy
"""
slug: str
_bundle: Optional[_GrapherBundle] = None
@property
def bundle(self) -> _GrapherBundle:
# LARS: give a nice error if the chart does not exist
if self._bundle is None:
self._bundle = _fetch_bundle(self.slug)
return self._bundle
@property
def config(self) -> dict:
return self.bundle.config # type: ignore
def get_data(self) -> pd.DataFrame:
return self.bundle.to_frame()
def __lt__(self, other):
return self.slug < other.slug
def __eq__(self, value: object) -> bool:
return isinstance(value, Chart) and value.slug == self.slug
def list_charts() -> List[str]:
"""
List all available charts published on Our World in Data.
"""
return sorted(_list_charts())
def get_data(slug: str) -> pd.DataFrame:
"""
Fetch the data for a chart by its slug.
"""
return Chart(slug).get_data()
`

export const getNotebookScript = (
slug: string
) => `import {StarboardEmbed} from "https://unpkg.com/starboard-wrap/dist/index.js"
const mount = document.querySelector("#notebook");
const notebook = \`# %% [markdown]
# Setup code
The code in the collapsed cell below provides the infrastructure to access our data. It is automatically run when the page is loaded and not very interesting to look at.
# %%--- [python]
# properties:
# collapsed: true
# run_on_load: true
# ---%%
${notebook}
# %% [markdown]
Add your code below - the example shows the first few rows of the data. Refer to the pandas documentation of ask ChatGPT for help on how to manipulate pandas dataframes.
# %%--- [python]
# properties:
# run_on_load: true
# ---%%
df = get_data("${slug}")
df.head()
\`
const el = new StarboardEmbed({
notebookContent: notebook,
src: "https://unpkg.com/starboard-notebook/dist/index.html"
});
mount.appendChild(el);
`

export const GrapherPage = (props: {
grapher: GrapherInterface
relatedCharts?: RelatedChart[]
Expand Down Expand Up @@ -411,7 +161,6 @@ window.Grapher.renderSingleGrapherOnGrapherPage(jsonConfig)`
</div>
)}
</main>
<div id="notebook" />
<SiteFooter
baseUrl={baseUrl}
context={SiteFooterContext.grapherPage}
Expand All @@ -420,12 +169,6 @@ window.Grapher.renderSingleGrapherOnGrapherPage(jsonConfig)`
type="module"
dangerouslySetInnerHTML={{ __html: script }}
/>
<script
type="module"
dangerouslySetInnerHTML={{
__html: getNotebookScript(grapher.slug!),
}}
/>
</body>
</html>
)
Expand Down

0 comments on commit 4a1192b

Please sign in to comment.