more work on new noun parsing, and inflections

.gitignore

@@ -10,6 +10,7 @@ lerna-debug.log*
 # fetched vocab
 src/verbs.ts
 src/nouns-adjs.ts
+vocab/mini-dict-entries.ts
 
 # testing
 /coverage

check-all-inflections.ts

@@ -15,7 +15,7 @@ type InflectionError = {
 
 async function checkAll() {
   console.log("Checking inflection functions on all dictionary words");
-  const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL);
+  const res = await fetch(process.env.LINGDOCS_DICTIONARY_URL + ".json");
   const { entries }: T.Dictionary = await res.json();
   const errors: InflectionError[] = [];
 

get-mini-dict.ts

@@ -0,0 +1,22 @@
+import * as T from "./src/types";
+import fs from "fs";
+
+import { entries as collection } from "./vocab/mini-dict-tss";
+
+const res = await fetch(
+  "https://storage.lingdocs.com/dictionary/dictionary.json"
+);
+const dictionary = (await res.json()) as T.Dictionary;
+
+const entries: T.DictionaryEntry[] = dictionary.entries.filter((x) =>
+  collection.includes(x.ts)
+);
+
+const contents = `import { DictionaryEntry } from "../src/types";
+// DO NOT MODIFY - GENERATED FROM mini-dict-tss.ts
+export const entries: DictionaryEntry[] = [
+${entries.map((e) => `\t${JSON.stringify(e)},`).join("\n")}
+];
+`;
+
+fs.writeFileSync("./vocab/mini-dict-entries.ts", contents);

jest.config.js

@@ -6,6 +6,7 @@ export default {
       "ts-jest",
       {
         tsconfig: "tsconfig.app.json",
+        diagnostics: false,
       },
     ],
   },

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pashto-inflector-website",
-  "version": "7.6.5",
+  "version": "7.7.0",
   "type": "module",
   "scripts": {
     "patch": "npm version patch --no-git-tag-version && cd src/lib && npm version patch --no-git-tag-version && cd ../components && npm version patch --no-git-tag-version",
@@ -14,12 +14,13 @@
     "build-website": "tsc -b && vite build",
     "build-components": "rm -rf src/components/dist && tsc --project src/components/tsconfig.json && cd src/components && node post-build.cjs",
     "build-lib": "rm -rf src/lib/dist && tsc --project src/lib/tsconfig.json && tsup src/lib/library.ts --format cjs && mv dist/library.cjs src/lib/dist/lib",
-    "get-words": "node get-words.cjs",
+    "get-words": "node get-words.cjs && tsx get-mini-dict.ts",
     "check-all-inflections": "tsx check-all-inflections.ts"
   },
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.15.2",
     "bootstrap": "4.6.1",
+    "json-edit-react": "^1.15.7",
     "react": "^18.3.1",
     "react-bootstrap": "1.5.1",
     "react-dom": "^18.3.1",

src/App.tsx

@@ -19,7 +19,7 @@ import { entryFeeder } from "./demo-components/entryFeeder";
 import Hider from "./components/src/Hider";
 import InflectionDemo from "./demo-components/InflectionDemo";
 import SpellingDemo from "./demo-components/SpellingDemo";
-import ParserDemo from "./demo-components/ParserDemo";
+// import ParserDemo from "./demo-components/ParserDemo";
 // import InflectionTable from "./components/src/InflectionsTable";
 
 function App() {
@@ -31,22 +31,21 @@ function App() {
     defualtTextOptions,
     "textOpts1"
   );
-  const [dictionaryReady, setDictionaryIsReady] = useState<boolean>(false);
   const [theme, setTheme] = useStickyState<"light" | "dark">("light", "theme1");
   const [showing, setShowing] = useState<string>("");
   function handleHiderClick(label: string) {
     setShowing((os) => (os === label ? "" : label));
   }
 
   useEffect(() => {
-    console.log("WILL INIT");
     dictionary
       .initialize()
-      .then(() => {
-        console.log("DONE INIT");
-        setDictionaryIsReady(true);
-      })
-      .catch(console.error);
+      .catch(console.error)
+      .then((res) => {
+        if (res && res.response === "loaded from saved") {
+          dictionary.update();
+        }
+      });
   }, []);
 
   useEffect(() => {
@@ -99,7 +98,6 @@ function App() {
             <h1 className="display-4 mt-2">
               <code>Pashto Inflector</code>
             </h1>
-            {dictionaryReady && <div>READY</div>}
             <p
               className="lead my-3"
               style={{ maxWidth: "600px", margin: "0 auto" }}
@@ -165,14 +163,18 @@ function App() {
           >
             <SpellingDemo opts={textOptions} onChange={setTextOptions} />
           </Hider>
-          <Hider
+          {/* <Hider
             label="Parser (🚧 IN PROGRESS 🚧)"
             hLevel={3}
             showing={showing === "parser"}
             handleChange={() => handleHiderClick("parser")}
           >
-            <ParserDemo opts={textOptions} entryFeeder={entryFeeder} />
-          </Hider>
+            <ParserDemo
+              opts={textOptions}
+              entryFeeder={entryFeeder}
+              dictionary={dictionary}
+            />
+          </Hider> */}
         </div>
       </main>
       <Modal

src/components/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lingdocs/ps-react",
-  "version": "7.6.5",
+  "version": "7.7.0",
   "description": "Pashto inflector library module with React components",
   "main": "dist/components/library.js",
   "module": "dist/components/library.js",

src/demo-components/ParserDemo.tsx

@@ -1,10 +1,13 @@
 import { useState } from "react";
 import * as T from "../types";
-import { parsePhrase } from "../lib/src/parsing/parse-phrase";
+// import { parsePhrase } from "../lib/src/parsing/parse-phrase";
 import { tokenizer } from "../lib/src/parsing/tokenizer";
-import { NPDisplay } from "../components/library";
-import EditableVP from "../components/src/vp-explorer/EditableVP";
-import { uncompleteVPSelection } from "../lib/src/phrase-building/vp-tools";
+// import { NPDisplay } from "../components/library";
+// import EditableVP from "../components/src/vp-explorer/EditableVP";
+// import { uncompleteVPSelection } from "../lib/src/phrase-building/vp-tools";
+import { DictionaryAPI } from "../lib/src/dictionary/dictionary";
+import { parseNoun } from "../lib/src/parsing/parse-noun-new";
+import { JsonEditor } from "json-edit-react";
 
 const working = [
   "limited demo vocab",
@@ -44,16 +47,17 @@ const examples = [
 ];
 
 function ParserDemo({
-  opts,
-  entryFeeder,
+  // opts,
+  // entryFeeder,
+  dictionary,
 }: {
   opts: T.TextOptions;
   entryFeeder: T.EntryFeeder;
+  dictionary: DictionaryAPI;
 }) {
   const [text, setText] = useState<string>("");
-  const [result, setResult] = useState<
-    ReturnType<typeof parsePhrase>["success"]
-  >([]);
+  const [result, setResult] = useState<any[]>([]);
+  // ReturnType<typeof parsePhrase>["success"]
   const [errors, setErrors] = useState<string[]>([]);
   function handleInput(value: string) {
     if (!value) {
@@ -62,7 +66,11 @@ function ParserDemo({
       setErrors([]);
       return;
     }
-    const { success, errors } = parsePhrase(tokenizer(value));
+    const res = parseNoun(tokenizer(value), dictionary, undefined, []);
+    const success = res.filter((x) => !x.tokens.length).map((x) => x.body);
+    const errors = [
+      ...new Set(res.flatMap(({ errors }) => errors.map((e) => e.message))),
+    ];
     setText(value);
     setErrors(errors);
     setResult(success);
@@ -127,8 +135,8 @@ function ParserDemo({
           <div className="text-center">Did you mean:</div>
         </>
       )}
-
-      {result.map((res) =>
+      <JsonEditor data={result} />
+      {/* {result.map((res) =>
         "inflected" in res ? (
           <NPDisplay NP={res.selection} inflected={res.inflected} opts={opts} />
         ) : "verb" in res ? (
@@ -166,7 +174,7 @@ function ParserDemo({
             <pre>{JSON.stringify(res, null, "  ")}</pre>
           </samp>
         )
-      )}
+      )} */}
       <details>
         <summary>AST</summary>
         <samp>

src/lib/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lingdocs/inflect",
-  "version": "7.6.5",
+  "version": "7.7.0",
   "description": "Pashto inflector library",
   "main": "dist/lib/library.cjs",
   "module": "dist/lib/library.js",

src/lib/src/dictionary/dictionary.ts

@@ -31,9 +31,44 @@ function nounLookup(p: string): T.NounEntry[] {
   return res.filter(tp.isNounEntry);
 }
 
-export const dictionary = {
+function otherLookup(
+  key: keyof T.DictionaryEntry,
+  p: string
+): T.DictionaryEntry[] {
+  if (!dictDb.collection) {
+    return [];
+  }
+  return dictDb.collection.find({ [key]: p });
+}
+
+function specialPluralLookup(p: string): T.NounEntry[] {
+  if (!dictDb.collection) {
+    return [];
+  }
+  const regex = new RegExp(`(^|\\s|,)${p}($|,)`);
+  return dictDb.collection
+    .find({
+      $or: [{ ppp: { $regex: regex } }, { app: { $regex: regex } }],
+    })
+    .filter(tp.isNounEntry);
+}
+
+export type DictionaryAPI = {
+  initialize: () => ReturnType<typeof dictDb.initialize>;
+  update: () => ReturnType<typeof dictDb.updateDictionary>;
+  queryP: (p: string) => T.DictionaryEntry[];
+  adjLookup: (p: string) => T.AdjectiveEntry[];
+  nounLookup: (p: string) => T.NounEntry[];
+  otherLookup: (key: keyof T.DictionaryEntry, p: string) => T.DictionaryEntry[];
+  specialPluralLookup: (p: string) => T.NounEntry[];
+};
+
+export const dictionary: DictionaryAPI = {
   initialize: async () => await dictDb.initialize(),
+  update: async () => await dictDb.updateDictionary(() => null),
   queryP: memoizedQueryP,
   adjLookup: memoize(adjLookup),
   nounLookup: memoize(nounLookup),
+  otherLookup: memoize(otherLookup),
+  specialPluralLookup: memoize(specialPluralLookup),
 };

src/lib/src/fp-ps.ts

@@ -53,6 +53,20 @@ export function fmapParseResult<A extends object, B extends object>(
   }));
 }
 
+export function fFlatMapParseResult<A extends object, B extends object>(
+  f: (x: A) => B[],
+  x: T.ParseResult<A>[]
+): T.ParseResult<B>[] {
+  return x.flatMap<T.ParseResult<B>>((xi) => {
+    const bodies = f(xi.body);
+    return bodies.map((body) => ({
+      tokens: xi.tokens,
+      body,
+      errors: xi.errors,
+    }));
+  });
+}
+
 export function fmapSingleOrLengthOpts<A, B>(
   f: (x: A) => B,
   x: T.SingleOrLengthOpts<A>
@@ -216,3 +230,35 @@ export function mapVerbRenderedOutput(
     return fmapVB(v);
   }
 }
+
+/**
+ * a type predicate OR combinator
+ */
+export function orTp<A, B extends A, C extends A>(
+  f: (x: A) => x is B,
+  g: (x: A) => x is C
+): (x: A) => x is B | C {
+  return (x: A) => f(x) || g(x);
+}
+
+/**
+ * a type predicate AND combinator
+ */
+export function andTp<A, B extends A, C extends A>(
+  f: (x: A) => x is B,
+  g: (x: A) => x is C
+): (x: A) => x is B & C {
+  return (x: A) => f(x) && g(x);
+}
+
+/**
+ * a type predicate successive AND combinator
+ * the second predicate is based on the first predicate
+ * being true and narrows the type further
+ */
+export function andSuccTp<A, B extends A, C extends B>(
+  f: (x: A) => x is B,
+  g: (x: B) => x is C
+): (x: A) => x is B & C {
+  return (x: A) => f(x) && g(x);
+}

src/lib/src/inflection-pattern.ts

@@ -13,6 +13,7 @@ import {
 export function getInflectionPattern(
   e: T.InflectableEntry
 ): T.InflectionPattern {
+  if (e.noInf) return 0;
   return isPattern1Entry(e)
     ? T.InflectionPattern.Basic
     : isPattern2Entry(e)