Skip to content

Commit

Permalink
fix(historical): map requests to chart() with deprec notice (#795)
Browse files Browse the repository at this point in the history 
Previously it used the "download" API which is no longer available.
See the above issue for more info.
  • Loading branch information
@gadicc
gadicc committed Sep 16, 2024
1 parent c131e5c commit c212df9
Show file tree
Hide file tree
Showing 27 changed files with 1,773 additions and 49 deletions.
108 changes: 63 additions & 45 deletions src/modules/historical.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,28 @@
import { jest } from "@jest/globals";

import historical from "./historical.js";
import chart from "./chart.js";
import testSymbols from "../../tests/testSymbols.js";

import testYf from "../../tests/testYf.js";
import { consoleSilent, consoleRestore } from "../../tests/console.js";

const yf = testYf({ historical });
const yf = testYf({ historical, chart });

describe("historical", () => {
// See also common module tests in moduleExec.spec.js

const symbols = testSymbols({
skip: ["BEKE", "BFLY", "SIMP", "^VXAPL", "APS.AX" /* Not Found */],
skip: [
"BEKE",
"BFLY",
"SIMP",
"^VXAPL",
"APS.AX", // Not Found
"ADH", // Not found
"SIX", // Not found
"SI", // Not found
],
});

it.each(symbols)("passes validation for symbol '%s'", async (symbol) => {
Expand All @@ -22,7 +32,7 @@ describe("historical", () => {
period1: "2020-01-01",
period2: "2020-01-03",
},
{ devel: `historical-${symbol}-2020-01-01-to-2020-01-03.json` },
{ devel: `historical-via-chart-${symbol}-2020-01-01-to-2020-01-03.json` },
);
});

Expand Down Expand Up @@ -50,7 +60,10 @@ describe("historical", () => {
period2: "2022-01-31",
events: "dividends",
},
{ devel: "historical-MSFT-dividends-2021-02-01-to-2022-01-31.csv" },
{
devel:
"historical-via-chart-MSFT-dividends-2021-02-01-to-2022-01-31.csv",
},
);
});

Expand All @@ -62,10 +75,12 @@ describe("historical", () => {
period2: "2022-01-31",
events: "split",
},
{ devel: "historical-NVDA-split-2021-02-01-to-2022-01-31.csv" },
{ devel: "historical-via-chart-NVDA-split-2021-02-01-to-2022-01-31.csv" },
);
});

/*
// Note: module no longer moduleExec, instead calls chart()
describe("transformWith", () => {
const yf = { _moduleExec: jest.fn(), historical };
// @ts-ignore: TODO
Expand All @@ -80,48 +95,51 @@ describe("historical", () => {
expect(options.period2).toBe(Math.floor(now.getTime() / 1000));
});
});
*/

// #208
describe("null values", () => {
if (process.env.FETCH_DEVEL !== "nocache")
it("strips all-null rows", async () => {
const createHistoricalPromise = () =>
yf.historical(
"EURGBP=X",
{
period1: 1567728000,
period2: 1570665600,
},
// Not a "fake" but seems fixed in newer Yahoo requests
// so let's test against our previously saved cache.
{ devel: "historical-EURGBP-nulls.saved.fake.json" },
);

await expect(createHistoricalPromise()).resolves.toBeDefined();

const result = await createHistoricalPromise();

// Without stripping, it's about 25 rows.
expect(result.length).toBe(5);

// No need to really check there are no nulls in the data, as
// validation handles that for us automatically.
});

if (process.env.FETCH_DEVEL !== "nocache")
it("throws on a row with some nulls", () => {
consoleSilent();
return expect(
yf
.historical(
if (false)
// Irrelevant for "via-chart"
describe("null values", () => {
if (process.env.FETCH_DEVEL !== "nocache")
it("strips all-null rows", async () => {
const createHistoricalPromise = () =>
yf.historical(
"EURGBP=X",
{ period1: 1567728000, period2: 1570665600 },
{ devel: "historical-EURGBP-nulls.fake.json" },
)
.finally(consoleRestore),
).rejects.toThrow("SOME (but not all) null values");
});
});
{
period1: 1567728000,
period2: 1570665600,
},
// Not a "fake" but seems fixed in newer Yahoo requests
// so let's test against our previously saved cache.
{ devel: "historical-EURGBP-nulls.saved.fake.json" },
);

await expect(createHistoricalPromise()).resolves.toBeDefined();

const result = await createHistoricalPromise();

// Without stripping, it's about 25 rows.
expect(result.length).toBe(5);

// No need to really check there are no nulls in the data, as
// validation handles that for us automatically.
});

if (process.env.FETCH_DEVEL !== "nocache")
it("throws on a row with some nulls", () => {
consoleSilent();
return expect(
yf
.historical(
"EURGBP=X",
{ period1: 1567728000, period2: 1570665600 },
{ devel: "historical-EURGBP-nulls.fake.json" },
)
.finally(consoleRestore),
).rejects.toThrow("SOME (but not all) null values");
});
});

it("handles events:dividends for stocks with no dividends (#658)", async () => {
const data = await yf.historical(
Expand All @@ -132,7 +150,7 @@ describe("historical", () => {
events: "dividends",
interval: "1d",
},
{ devel: "historical-dividends-TSLA-no-dividends.json" },
{ devel: "historical-via-chart-dividends-TSLA-no-dividends.json" },
);
// Enough to check that this doesn't throw.
});
Expand Down
99 changes: 95 additions & 4 deletions src/modules/historical.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,15 @@
import { Static, Type } from "@sinclair/typebox";
import { Value } from "@sinclair/typebox/value";
import type {
ModuleOptions,
ModuleOptionsWithValidateTrue,
ModuleOptionsWithValidateFalse,
ModuleThis,
} from "../lib/moduleCommon.js";
import { YahooFinanceDate, YahooNumber } from "../lib/yahooFinanceTypes.js";
import _chart, { ChartOptionsSchema } from "./chart.js";
import validateAndCoerceTypebox from "../lib/validateAndCoerceTypes.js";
import { showNotice } from "../lib/notices.js";

const HistoricalRowHistory = Type.Object(
{
Expand Down Expand Up @@ -52,7 +56,14 @@ const HistoricalOptionsSchema = Type.Object(
Type.Literal("1mo"),
]),
),
events: Type.Optional(Type.String()),
// events: Type.Optional(Type.String()),
events: Type.Optional(
Type.Union([
Type.Literal("history"),
Type.Literal("dividends"),
Type.Literal("split"),
]),
),
includeAdjustedClose: Type.Optional(Type.Boolean()),
},
{ title: "HistoricalOptions" },
Expand Down Expand Up @@ -149,12 +160,21 @@ export default function historical(
moduleOptions?: ModuleOptionsWithValidateFalse,
): Promise<any>;

export default function historical(
export default async function historical(
this: ModuleThis,
symbol: string,
queryOptionsOverrides: HistoricalOptions,
moduleOptions?: ModuleOptions,
): Promise<any> {
showNotice("ripHistorical");

validateAndCoerceTypebox({
type: "options",
data: queryOptionsOverrides ?? {},
schema: HistoricalOptionsSchema,
options: this._opts.validation,
});

let schema;
if (
!queryOptionsOverrides.events ||
Expand All @@ -167,6 +187,76 @@ export default function historical(
schema = HistoricalStockSplitsResultSchema;
else throw new Error("No such event type:" + queryOptionsOverrides.events);

const queryOpts = { ...queryOptionsDefaults, ...queryOptionsOverrides };
if (!Value.Check(HistoricalOptionsSchema, queryOpts))
throw new Error(
"Internal error, please report. Overrides validated but not defaults?",
);

// Don't forget that queryOpts are already validated and safe-safe.
const eventsMap = { history: "", dividends: "div", split: "split" };
const chartQueryOpts = {
period1: queryOpts.period1,
period2: queryOpts.period2,
interval: queryOpts.interval,
events: eventsMap[queryOpts.events || "history"],
};
if (!Value.Check(ChartOptionsSchema, chartQueryOpts))
throw new Error(
"Internal error, please report. historical() provided invalid chart() query options.",
);

// TODO: do we even care?
if (queryOpts.includeAdjustedClose === false) {
/* */
}

const result = await (this.chart as typeof _chart)(symbol, chartQueryOpts, {
...moduleOptions,
validateResult: true,
});

let out;
if (queryOpts.events === "dividends") {
out = (result.events?.dividends ?? []).map((d) => ({
date: d.date,
dividends: d.amount,
}));
} else if (queryOpts.events === "split") {
out = (result.events?.splits ?? []).map((s) => ({
date: s.date,
stockSplits: s.splitRatio,
}));
} else {
out = result.quotes;
}

const validateResult =
!moduleOptions ||
moduleOptions.validateResult === undefined ||
moduleOptions.validateResult === true;

const validationOpts = {
...this._opts.validation,
// Set logErrors=false if validateResult=false
logErrors: validateResult ? this._opts.validation.logErrors : false,
};

try {
validateAndCoerceTypebox({
type: "result",
data: out,
schema,
options: validationOpts,
});
} catch (error) {
if (validateResult) throw error;
}

return out;

/*
// Original historical() retrieval code when Yahoo API still existed.
return this._moduleExec({
moduleName: "historical",
Expand Down Expand Up @@ -237,7 +327,7 @@ export default function historical(
if (nullCount === 0) {
// No nulls is a legit (regular) result
filteredResults.push(row);
} else if (nullCount !== fieldCount - 1 /* skip "date" */) {
} else if (nullCount !== fieldCount - 1 /* skip "date" */ /*) {
// Unhandled case: some but not all values are null.
// Note: no need to check for null "date", validation does it for us
console.error(nullCount, row);
Expand All @@ -255,11 +345,12 @@ export default function historical(
* We may consider, for future optimization, to count rows and create
* new array in advance, and skip consecutive blocks of null results.
* Of doubtful utility.
*/
*/ /*
return filteredResults;
},
},
moduleOptions,
});
*/
}
64 changes: 64 additions & 0 deletions tests/http/historical-via-chart-0P000071W8.TO-2020-01-01-to-2020-01-03.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
{
"request": {
"url": "https://query2.finance.yahoo.com/v8/finance/chart/0P000071W8.TO?useYfid=true&interval=1d&includePrePost=true&events=&lang=en-US&period1=1577836800&period2=1578009600"
},
"response": {
"ok": true,
"status": 200,
"statusText": "OK",
"headers": {
"content-type": [
"application/json;charset=utf-8"
],
"y-rid": [
"6a6h2g1jeg8bt"
],
"cache-control": [
"public, max-age=10, stale-while-revalidate=20"
],
"vary": [
"Origin,Accept-Encoding"
],
"content-length": [
"1059"
],
"x-envoy-upstream-service-time": [
"3"
],
"date": [
"Mon, 16 Sep 2024 12:14:24 GMT"
],
"server": [
"ATS"
],
"x-envoy-decorator-operation": [
"finance-chart-api--mtls-production-ir2.finance-k8s.svc.yahoo.local:4080/*"
],
"age": [
"93"
],
"strict-transport-security": [
"max-age=31536000"
],
"referrer-policy": [
"no-referrer-when-downgrade"
],
"x-frame-options": [
"SAMEORIGIN"
],
"connection": [
"close"
],
"expect-ct": [
"max-age=31536000, report-uri=\"http://csp.yahoo.com/beacon/csp?src=yahoocom-expect-ct-report-only\""
],
"x-xss-protection": [
"1; mode=block"
],
"x-content-type-options": [
"nosniff"
]
},
"body": "{\"chart\":{\"result\":[{\"meta\":{\"currency\":\"CAD\",\"symbol\":\"0P000071W8.TO\",\"exchangeName\":\"TOR\",\"fullExchangeName\":\"Toronto\",\"instrumentType\":\"MUTUALFUND\",\"firstTradeDate\":1514903400,\"regularMarketTime\":1726171200,\"hasPrePostMarketData\":false,\"gmtoffset\":-14400,\"timezone\":\"EDT\",\"exchangeTimezoneName\":\"America/Toronto\",\"regularMarketPrice\":133.45,\"longName\":\"TD US Index e\",\"shortName\":\"TD U.S. Index Fund - e\\\",\",\"chartPreviousClose\":73.06,\"priceHint\":2,\"currentTradingPeriod\":{\"pre\":{\"timezone\":\"EDT\",\"start\":1726488000,\"end\":1726493400,\"gmtoffset\":-14400},\"regular\":{\"timezone\":\"EDT\",\"start\":1726493400,\"end\":1726516800,\"gmtoffset\":-14400},\"post\":{\"timezone\":\"EDT\",\"start\":1726516800,\"end\":1726520400,\"gmtoffset\":-14400}},\"dataGranularity\":\"1d\",\"range\":\"\",\"validRanges\":[\"1mo\",\"3mo\",\"6mo\",\"ytd\",\"1y\",\"2y\",\"5y\",\"10y\",\"max\"]},\"timestamp\":[1577975400],\"indicators\":{\"quote\":[{\"open\":[73.72000122070312],\"close\":[73.72000122070312],\"low\":[73.72000122070312],\"high\":[73.72000122070312],\"volume\":[0]}],\"adjclose\":[{\"adjclose\":[73.72000122070312]}]}}],\"error\":null}}"
}
}
Loading

0 comments on commit c212df9

Please sign in to comment.