Create a GPU accelerated examples/extra-large-graphs

examples/extra-large-graphs/README.md

@@ -0,0 +1,105 @@
+# Extra Large Graphs Demo with `@rapidsai/node` GPU acceleration
+
+RAPIDS.ai is an open source GPU-acceleration project. We're building new
+tools with familiar APIs and extending existing tools so that more
+scientists and users can take advantage of GPU performance.
+
+This project creates a `RapidsGraphologyGraph` subclass of `Graph`,
+plus a modified `index.ts` to make GPU-stored graph vertices and edges
+available to your browser session. It talks to the `@rapidai/demo-api-server`
+that has been built using [node-rapids](https://www.github.com/rapidsai/node).
+
+# Performance
+
+Using a GPU for graph rendering offers substantial performance increases.
+In this demo, I've included screenshots of performance comparisons between
+using our GPU-backed `RapidsGraphologyGraph` and the original renderer
+included in the `large-graphs` demo.
+
+When the complexity of the graph matches the original demo, performance
+is comparable:
+
+![original size of graph, screenshot](./gpu_performance_same.png)
+
+When the complexity increases substantially, the GPU performance improvement
+is marked:
+
+![size increased by 100x](./gpu_performance_two_oom_bigger.png)
+
+It is important to note in the larger case that the "I/O" benchmark for
+the original `large-graphs` demo includes the graph generation time, which
+is substantial. However, were the file stored in a 500MB .json (as in the
+GPU case) and parsed in-browser, execution time is similar or longer.
+
+# What Is Happening
+
+In order to run this demo, you need the `@rapidsai/demo` npm package,
+a system with a fairly modern NVIDIA GPU, and have previously installed
+the [CUDA Toolkit] (https://developer.nvidia.com/cuda-toolkit).
+
+The node-rapids workspace [demo-api-server](https://github.com/rapidsai/node)
+is available as a backend to any HTTP client. At this time only limited
+functionality is available to parse JSON files in the `graphology`
+graph dataset format, plus API requests to request Dataframes and
+their Columns via `apache-arrow`.
+
+Two endpoints, `graphology/nodes` and `graphology/edges` specifically
+return pre-formatted arrays that can be used directly with the
+[sigma.js](https://github.com/jacomyal/sigma.js) renderer.
+
+## Additional Dependencies
+
+- @rapidsai/demo-api-server
+- apache-arrow
+
+## To run the demo
+
+Due to native dependency distribution complexity, pre-packaged builds of
+the node-rapids modules are presently only available via our [public docker images](https://github.com/orgs/rapidsai/packages/container/package/node).
+See [USAGE.md](https://github.com/rapidsai/node/tree/main/USAGE.md) for more details.
+
+Generate a graph of your liking, or provide another:
+
+```bash
+cd $SIGMAJSHOME/examples/extra-large-graphs
+node generate-graph.js 10000 20000 3 graphology.json
+```
+
+Run `@rapidsai/demo-api-server` via docker:
+
+```bash
+REPO=ghcr.io/rapidsai/node
+VERSIONS="22.06.00-runtime-node16.15.1-cuda11.6.2-ubuntu20.04"
+
+# Be sure to pass either the `--runtime=nvidia` or `--gpus` flag!
+docker run --rm \
+    --runtime=nvidia \
+    -e "DISPLAY=$DISPLAY" \
+    -v "/etc/fonts:/etc/fonts:ro" \
+    -v "/tmp/.X11-unix:/tmp/.X11-unix:rw" \
+    -v "/usr/share/fonts:/usr/share/fonts:ro" \
+    -v "/usr/share/icons:/usr/share/icons:ro" \
+    -v "$(realpath -m ./graphology.json):/home/node/node_modules/@rapidsai/demo-api-server/public/graphology.json"
+    $REPO:$VERSIONS-demo \
+    npx @rapidsai/demo-api-server
+```
+
+We expect to have full `npm` support soon.
+
+Finally run the `extra-large-graphs` demo in the normal fashion:
+
+```bash
+cd $SIGMAJSROOT/examples
+npm start --example=extra-large-graphs
+```  
+
+## Dataset
+
+Run the graph generator at <https://github.com/thomcom/sigma.js/blob/add-gpu-graph-to-example/examples/extra-large-graphs/generate-graph.js>
+to create a very large graph using the command:
+
+```bash
+node graph-generator.js 1000000 2000000 3 graphology.json
+```
+Any dataset that matches the schema of `graphology.json` is supported.
+

examples/extra-large-graphs/edge.gpu.ts

@@ -0,0 +1,91 @@
+/**
+ * Sigma.js WebGL Renderer Fast Edge Program
+ * ==========================================
+ *
+ * Program rendering edges using GL_LINES which is presumably very fast but
+ * won't render thickness correctly on some GPUs and has some quirks.
+ * @module
+ */
+import { AbstractEdgeProgram }  from "sigma/rendering/webgl/programs/common/edge";
+import { RenderParams } from "sigma/rendering/webgl/programs/common/program";
+
+import fragmentShaderSource from "sigma/rendering/webgl/shaders/edge.fast.frag.glsl";
+import vertexShaderSource from "sigma/rendering/webgl/shaders/edge.fast.vert.glsl";
+
+let POINTS = 2;
+const ATTRIBUTES = 3;
+
+export default class EdgeGpuProgram extends AbstractEdgeProgram {
+  positionLocation: GLint;
+  colorLocation: GLint;
+  matrixLocation: WebGLUniformLocation;
+
+  constructor(gl: WebGLRenderingContext) {
+    super(gl, vertexShaderSource, fragmentShaderSource, POINTS, ATTRIBUTES);
+
+    // Locations:
+    this.positionLocation = gl.getAttribLocation(this.program, "a_position");
+    this.colorLocation = gl.getAttribLocation(this.program, "a_color");
+
+    // Uniform locations:
+    const matrixLocation = gl.getUniformLocation(this.program, "u_matrix");
+    if (matrixLocation === null) throw new Error("EdgeGpuProgram: error while getting matrixLocation");
+    this.matrixLocation = matrixLocation;
+
+    this.bind();
+  }
+
+  bind(): void {
+    const gl = this.gl;
+
+    // Bindings
+    gl.enableVertexAttribArray(this.positionLocation);
+    gl.enableVertexAttribArray(this.colorLocation);
+
+    gl.vertexAttribPointer(
+      this.positionLocation,
+      2,
+      gl.FLOAT,
+      false,
+      this.attributes * Float32Array.BYTES_PER_ELEMENT,
+      0,
+    );
+    gl.vertexAttribPointer(
+      this.colorLocation,
+      4,
+      gl.UNSIGNED_BYTE,
+      true,
+      this.attributes * Float32Array.BYTES_PER_ELEMENT,
+      8,
+    );
+  }
+
+  computeIndices(): void {
+    //nothing to do
+  }
+
+  process(
+    sourceData,
+    targetData,
+    data,
+    hidden: boolean,
+    offset: number,
+  ): void {
+    this.array = data.buffer.getChild('edges').toArray();
+    POINTS = this.array.length / ATTRIBUTES;
+    this.points = POINTS;
+  }
+
+  render(params: RenderParams): void {
+    if (this.hasNothingToRender()) return;
+
+    const gl = this.gl;
+    const program = this.program;
+
+    gl.useProgram(program);
+
+    gl.uniformMatrix3fv(this.matrixLocation, false, params.matrix);
+
+    gl.drawArrays(gl.LINES, 0, this.array.length / ATTRIBUTES);
+  }
+}

examples/extra-large-graphs/generate-graph.js

@@ -0,0 +1,69 @@
+const fs = require('fs');
+const seedrandom = require('seedrandom');
+
+const {Graph} = require('graphology');
+
+const circlepack = require( 'graphology-layout/circlepack');
+const clusters = require( 'graphology-generators/random/clusters');
+
+try {
+  const args = process.argv.slice(2);
+  const order = Number(args[0]);
+  const size = Number(args[1]);
+  const num_clusters = Number(args[2]);
+  const outputName = args[3];
+  const rng = seedrandom('sigma');
+  const state = {
+    size: size,
+    order: order,
+    clusters: num_clusters,
+    edgesRenderer: 'edges-fast'
+  };
+
+  // 1. Generate a graph:
+  const graph = clusters(Graph, { ...state, rng });
+  circlepack.assign(graph, {
+    hierarchyAttributes: ['cluster'],
+  });
+  const colors = {};
+  for (let i = 0; i < +state.clusters; i++) {
+    colors[i] = '#' + Math.floor(rng() * 16777215).toString(16);
+  }
+  let i = 0;
+  graph.forEachNode((node, { cluster }) => {
+    graph.mergeNodeAttributes(node, {
+      size: graph.degree(node) > 1 ? graph.degree(node) / 2 : 1,
+      label: `Node n°${++i}, in cluster n°${cluster}`,
+      color: colors[cluster + ''],
+    });
+  });
+
+  // 2. Save the graph:
+  fs.writeFileSync(outputName, JSON.stringify(graph.toJSON(), {}, 2), (err) => {
+    if(err) {
+      console.log('unable to save file');
+    } else {
+      console.log('file saved');
+    }
+  });
+  console.log('Graph Generated.');
+  console.log('You must copy the output file to');
+  console.log('');
+  console.log('node/modules/demo/api-server/routes/public/graphology.json');
+  console.log('to use with node-rapids/demo-api-server');
+  console.log('and sigma.js/examples/extra-large-graph');
+}
+catch (e) {
+  console.log('Exception');
+  console.log(e);
+  console.log('');
+  console.log('generate-graph.js usage:');
+  console.log('------------------------');
+  console.log('');
+  console.log('node generate-graph.js <node_count> <edge_count> <clusters_count> <filename>');
+  console.log('');
+  console.log('example: ');
+  console.log('node generate-graph.js 2000000 1000000 3 graphology.json');
+  console.log('');
+}
+

examples/extra-large-graphs/gpu-loader.ts

@@ -0,0 +1,61 @@
+/**
+ * This example aims at showcasing sigma's performances.
+ */
+
+import arrow from 'apache-arrow';
+
+async function request(obj) {
+  return new Promise((resolve, reject) => {
+    let xhr = new XMLHttpRequest();
+    xhr.open(obj.method || 'GET', obj.url || obj);
+    if (obj.headers) {
+      Object.keys(obj.headers).forEach(key => { xhr.setRequestHeader(key, obj.headers[key]); });
+    }
+    xhr.onload = () => {
+      try {
+        if (xhr.status >= 200 && xhr.status < 300) {
+          resolve(xhr.response);
+        } else {
+          reject(xhr.statusText);
+        }
+      } catch (e) { reject(e); }
+    };
+    xhr.onerror = () => reject(xhr.statusText);
+    xhr.send(obj.body);
+  });
+}
+
+const SERVER             = 'http://localhost:3010';
+const DATASET_ROUTE      = '/graphology/read_large_demo?filename=../../public/graphology.json';
+const NODES_ROUTE        = '/graphology/get_column/nodes';
+const NODES_BOUNDS_ROUTE = '/graphology/nodes/bounds';
+const NODES_BUFFER_ROUTE = '/graphology/nodes';
+const EDGES_BUFFER_ROUTE = '/graphology/edges';
+const TABLE_ROUTE        = '/graphology/get_table';
+
+const GpuLoader = {
+  init: async ()          => request({method: 'POST', url: SERVER + DATASET_ROUTE, mode: 'no-cors'}),
+  getTable: async (table) => {
+    const result = await fetch(SERVER + TABLE_ROUTE + '/' + table, {method: 'GET', headers: {'Access-Control-Allow-Origin': '*'}})
+    return arrow.tableFromIPC(result);
+  },
+  getColumn: async (table, column) => {
+    const table_route  = {'nodes': '/graphology/get_column/nodes/'}[table];
+    const column_route = SERVER + table_route + column;
+    const result = await fetch(column_route, {method: 'GET', headers: {'Access-Control-Allow-Origin': '*'}});
+    return arrow.tableFromIPC(result);
+  },
+  getNodesBounds: async () => request(SERVER + NODES_BOUNDS_ROUTE),
+  getNodesBuffer: async () => {
+    const route = SERVER + NODES_BUFFER_ROUTE;
+    const result = await fetch(route, {method: 'GET', headers: {'Access-Control-Allow-Origin': '*'}});
+    return arrow.tableFromIPC(result);
+  },
+  getEdgesBuffer: async () => {
+    const route = SERVER + EDGES_BUFFER_ROUTE;
+    const result = await fetch(route, {method: 'GET', headers: {'Access-Control-Allow-Origin': '*'}});
+    return arrow.tableFromIPC(result);
+  }
+};
+
+export default GpuLoader;