Sigma.js demo using node-rapids

lerna.json

@@ -15,6 +15,7 @@
     "modules/demo/ssr/*",
     "modules/demo/tfjs/*",
     "modules/demo/client-server",
+    "modules/demo/api-server",
     "modules/demo/viz-app",
     "modules/demo/sql/*"
   ]

modules/cudf/src/column/from_arrow.ts

@@ -166,7 +166,7 @@ class DataToColumnVisitor extends arrow.Visitor {
   }
   visitStruct<T extends arrow.Struct>(data: arrow.Data<T>) {
     const {type, length, nullBitmap: nullMask} = data;
-    const children = type.children.map((_, i) => this.visit(data.children[i] ));
+    const children = type.children.map((_, i) => this.visit(data.children[i]));
     return new Column({
       length,
       type: new Struct(children.map(

modules/cudf/src/series.ts

@@ -1220,7 +1220,7 @@ export class AbstractSeries<T extends DataType = any> {
    * Copy the underlying device memory to host, and return an Iterator of the values.
    */
   [Symbol.iterator](): IterableIterator<T['TValue']|null> {
-    return this.toArrow()[Symbol.iterator]() ;
+    return this.toArrow()[Symbol.iterator]();
   }
 
   /**

modules/cudf/test/series/list-tests.ts

@@ -46,7 +46,7 @@ describe('ListSeries', () => {
 
   test('Can create from Arrow', () => {
     const vec  = listsOfInt32s([[0, 1, 2], [3, 4, 5]]);
-    const ints = vec.getChildAt<arrow.Int32>(0)! ;
+    const ints = vec.getChildAt<arrow.Int32>(0)!;
     const col  = Series.new(vec);
 
     validateOffsets(vec, col);
@@ -75,8 +75,8 @@ describe('ListSeries', () => {
 
   test('Can create a List of Lists from Arrow', () => {
     const vec  = listsOfListsOfInt32s([[[0, 1, 2]], [[3, 4, 5], [7, 8, 9]]]);
-    const list = vec.getChildAt<ListOfInt32>(0)! ;
-    const ints = list.getChildAt<arrow.Int32>(0)! ;
+    const list = vec.getChildAt<ListOfInt32>(0)!;
+    const ints = list.getChildAt<arrow.Int32>(0)!;
     const col  = Series.new(vec);
 
     validateOffsets(vec, col);
@@ -86,8 +86,8 @@ describe('ListSeries', () => {
 
   test('Can gather a List of Lists', () => {
     const vec  = listsOfListsOfInt32s([[[0, 1, 2]], [[3, 4, 5], [7, 8, 9]]]);
-    const list = vec.getChildAt<ListOfInt32>(0)! ;
-    const ints = list.getChildAt<arrow.Int32>(0)! ;
+    const list = vec.getChildAt<ListOfInt32>(0)!;
+    const ints = list.getChildAt<arrow.Int32>(0)!;
     const col  = Series.new(vec);
     const out  = col.gather(Series.new({type: new Int32, data: new Int32Array([0, 1, 2])}));
 

modules/cudf/test/series/struct-tests.ts

@@ -38,8 +38,8 @@ describe('StructSeries', () => {
       {x: 1, y: 4},
       {x: 2, y: 5},
     ]);
-    const xs  = vec.getChildAt<arrow.Int32>(0)! ;
-    const ys  = vec.getChildAt<arrow.Int32>(1)! ;
+    const xs  = vec.getChildAt<arrow.Int32>(0)!;
+    const ys  = vec.getChildAt<arrow.Int32>(1)!;
 
     const col = Series.new(vec);
 
@@ -53,9 +53,9 @@ describe('StructSeries', () => {
       {point: {x: 1, y: 4}},
       {point: {x: 2, y: 5}},
     ]);
-    const points = vec.getChildAt<StructOfInt32>(0)! ;
-    const xs     = points.getChildAt<arrow.Int32>(0)! ;
-    const ys     = points.getChildAt<arrow.Int32>(1)! ;
+    const points = vec.getChildAt<StructOfInt32>(0)!;
+    const xs     = points.getChildAt<arrow.Int32>(0)!;
+    const ys     = points.getChildAt<arrow.Int32>(1)!;
     const col    = Series.new(vec);
 
     validateElements(xs, col.getChild('point').getChild('x'));
@@ -74,9 +74,9 @@ describe('StructSeries', () => {
     expect(out.type.children[0].type.children[0].name).toEqual('x');
     expect(out.type.children[0].type.children[1].name).toEqual('y');
 
-    const points = vec.getChildAt<StructOfInt32>(0)! ;
-    const xs     = points.getChildAt<arrow.Int32>(0)! ;
-    const ys     = points.getChildAt<arrow.Int32>(1)! ;
+    const points = vec.getChildAt<StructOfInt32>(0)!;
+    const xs     = points.getChildAt<arrow.Int32>(0)!;
+    const ys     = points.getChildAt<arrow.Int32>(1)!;
 
     validateElements(xs, col.getChild('point').getChild('x'));
     validateElements(ys, col.getChild('point').getChild('y'));

modules/demo/.vscode/launch.json

@@ -13,14 +13,27 @@
         }
     ],
     "configurations": [
+        {
+            "name": "Attach Test",
+            "port": 9229,
+            "request": "attach",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node"
+        },
         {
             "type": "node",
             "request": "launch",
             "name": "Debug Demo (TS only)",
             "program": "${workspaceFolder}/${input:DEMO_NAME}",
             "stopOnEntry": false,
-            "args": ["${input:DEMO_ARGS}"],
-            "runtimeArgs": ["--experimental-vm-modules"],
+            "args": [
+                "${input:DEMO_ARGS}"
+            ],
+            "runtimeArgs": [
+                "--experimental-vm-modules"
+            ],
             "cwd": "${workspaceFolder}/${input:DEMO_NAME}",
             "console": "integratedTerminal",
             "internalConsoleOptions": "neverOpen",
@@ -44,7 +57,9 @@
         {
             "name": "Debug Demo (launch gdb)",
             // hide the individual configurations from the debug dropdown list
-            "presentation": { "hidden": true },
+            "presentation": {
+                "hidden": true
+            },
             "type": "cppdbg",
             "request": "launch",
             "stopAtEntry": false,
@@ -62,9 +77,18 @@
             ],
             "program": "${input:NODE_BINARY}",
             "environment": [
-                { "name": "NODE_DEBUG", "value": "1" },
-                { "name": "NODE_NO_WARNINGS", "value": "1" },
-                { "name": "NODE_ENV", "value": "production" },
+                {
+                    "name": "NODE_DEBUG",
+                    "value": "1"
+                },
+                {
+                    "name": "NODE_NO_WARNINGS",
+                    "value": "1"
+                },
+                {
+                    "name": "NODE_ENV",
+                    "value": "production"
+                },
                 // { "name": "READABLE_STREAM", "value": "disable" },
             ],
             "args": [
@@ -80,7 +104,9 @@
             "type": "node",
             "request": "attach",
             // hide the individual configurations from the debug dropdown list
-            "presentation": { "hidden": true },
+            "presentation": {
+                "hidden": true
+            },
             "port": 9229,
             "timeout": 60000,
             "cwd": "${workspaceFolder}",
@@ -113,7 +139,7 @@
             "command": "shellCommand.execute",
             "args": {
                 "description": "Select a demo to debug",
-                "command": "echo client-server viz-app luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
+                "command": "echo client-server viz-app api-server luma graph spatial xterm $(find modules/demo/deck modules/demo/tfjs modules/demo/ipc modules/demo/ssr modules/demo/sql -maxdepth 2 -type f -name 'package.json' -print0 | grep -z -v node_modules | tr -d '\\0' | sed -r 's@modules/demo/@@g' | sed -r 's@/package.json@ @g') | sort -Vr | sed -r 's@\\s@\\n@g'",
             }
         },
     ]

modules/demo/api-server/.gitignore

@@ -0,0 +1,58 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# node-waf configuration
+.lock-wscript
+
+# Compiled binary addons (http://nodejs.org/api/addons.html)
+build/Release
+
+# Dependency directories
+node_modules
+jspm_packages
+
+# Optional npm cache directory
+.npm
+
+# Optional REPL history
+.node_repl_history
+
+# 0x
+profile-*
+
+# mac files
+.DS_Store
+
+# vim swap files
+*.swp
+
+# webstorm
+.idea
+
+# vscode
+.vscode
+*code-workspace
+
+# clinic
+profile*
+*clinic*
+*flamegraph*

modules/demo/api-server/README.md

@@ -0,0 +1,87 @@
+# Fastify HTTP Server Demo
+
+This project is a Fastify-based node http server that allows
+commands and data to be sent to and from the NVIDIA GPUs installed
+on the host machine.
+
+Essentially, the node-rapids system is provided as a backend to any
+HTTP client. At this time only limited functionality is available to
+load JSON files in the `graphology` graph dataset format, plus API
+requests to request Dataframes and their Columns via `apache-arrow`.
+
+Two endpoints, `graphology/nodes` and `graphology/edges` specifically
+return pre-formatted arrays that can be used directly with the
+[sigma.js](https://github.com/jacomyal/sigma.js) renderer. An
+[extra-large-graphs](https://github.com/jacomyal/sigma.js/pull/1252) example PR is in the works
+that utilizes this GPU-accelerated data for rendering larger datasets
+than available via only CPU.
+
+## Main Dependencies
+- @rapidsai/cudf
+- fastify
+- fastify-arrow
+- apache-arrow
+
+An example project that demonstrates this API has a PR being reviewed at [sigma.js](https://github.com/jacomyal/sigma.js),
+but this project does not depend on sigma.js.j
+
+## Installation
+
+To install dependencies, run the following from the root directory for `node-rapids`
+
+```bash
+yarn
+```
+
+To run the demo
+```bash
+# Select the api-server demo from the list of demos
+yarn demo
+# OR specifically with
+cd modules/demo/api-server
+yarn start
+```
+
+## Dataset
+
+Run the graph generator at https://github.com/thomcom/sigma.js/blob/add-gpu-graph-to-example/examples/extra-large-graphs/generate-graph.js
+to create a very large graph using the object
+
+```js
+const state = {
+  order: 1000000,
+  size: 2000000,
+  clusters: 3,
+  edgesRenderer: 'edges-fast'
+};
+```
+
+You don't need to edit the file in order to create a graph of the above size. Simply call the .js via node:
+
+```bash
+node graph-generator.js
+```
+
+Which will create a file `./large-graph.json`. Copy `./large-graph.json` into `api-server/` and then set your
+API request to the location of the file relative to `routes/graphology/index.js`:
+
+```
+curl http://localhost:3000/graphology/read_large_demo?filename=../../large-graph.json
+```
+
+Which will use parallel JSON parsing to load the graph onto the GPU.
+
+## Routes
+
+```txt
+/
+/graphology
+/graphology/read_json
+/graphology/read_large_demo
+/graphology/list_tables
+/graphology/get_table/:table
+/graphology/get_column/:table/:column
+/graphology/nodes/bounds
+/graphology/nodes
+/graphology/edges
+/graphology/release

modules/demo/api-server/app.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const path     = require('path')
+const AutoLoad = require('@fastify/autoload')
+
+module.exports = async function(fastify, opts) {
+  // Place here your custom code!
+
+  // Do not touch the following lines
+
+  // This loads all plugins defined in plugins
+  // those should be support plugins that are reused
+  // through your application
+  fastify.register(AutoLoad,
+                   {dir: path.join(__dirname, 'plugins'), options: Object.assign({}, opts)})
+
+  // This loads all plugins defined in routes
+  // define your routes in one of these
+  fastify.register(
+    AutoLoad,
+    {dir: path.join(__dirname, 'routes'), options: Object.assign({}, opts), ignorePattern: /.*.ts/})
+}

modules/demo/api-server/index.js

@@ -0,0 +1,33 @@
+#!/usr/bin/env node
+
+// Copyright (c) 2022, NVIDIA CORPORATION.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const Path = require('path');
+
+// Change cwd to the example dir so relative file paths are resolved
+process.chdir(__dirname);
+
+const fastify = require.resolve('fastify-cli/cli.js');
+
+const {spawnSync} = require('child_process');
+
+const env = {
+  NEXT_TELEMETRY_DISABLED: 1,  // disable https://fastifyjs.org/telemetry
+  ...process.env,
+};
+
+spawnSync(process.execPath,
+          [fastify, 'start', '-l', 'info', '-P', '-w', 'app.js'],
+          {env, cwd: __dirname, stdio: 'inherit'});