{ "cells": [ { "cell_type": "markdown", "id": "8cbd794e", "metadata": {}, "source": [ "# Using Apache Arrow Flight with Rocketgraph xGT\n", "\n", "The outline of this notebook:\n", "\n", "1. Generate a table of edges with random data.\n", "1. Bring edge table into an Arrow format.\n", "1. Send Arrow Flight to Rocketgraph xGT.\n", "1. Request to run a graph query on xGT.\n", "1. Bring results back as an Arrow Flight.\n", "1. Visualize results." ] }, { "cell_type": "markdown", "id": "d639cf4a", "metadata": {}, "source": [ "## Generate edges with random data.\n", "\n", "First, set up parameters for edge table.\n", "\n", "The table will have four columns.\n", "The first two columns representing *source* and *target* of each edge.\n", "The other two columns representing edge attributes.\n", "\n", "The values of the first two columns are drawn randomly in the range of `1` to `NUM_NODES`." ] }, { "cell_type": "code", "execution_count": 1, "id": "a2b1bddc", "metadata": {}, "outputs": [], "source": [ "# Metadata of random data table\n", "NUM_ROWS=100\n", "NUM_NODES=10" ] }, { "cell_type": "code", "execution_count": 2, "id": "972db386", "metadata": {}, "outputs": [], "source": [ "# Generate table\n", "import pandas as pd\n", "import random\n", "random.seed()\n", "alpha='abcdefghijklmnopqrstuvwxyz'\n", "\n", "data = {\n", " 'source': [random.randint(1, NUM_NODES) for _ in range(NUM_ROWS)],\n", " 'target': [random.randint(1, NUM_NODES) for _ in range(NUM_ROWS)],\n", " 'attr1' : [random.choice(alpha) for _ in range(NUM_ROWS)],\n", " 'attr2' : [random.choice(alpha) for _ in range(NUM_ROWS)],\n", "}\n", "df = pd.DataFrame(data)" ] }, { "cell_type": "markdown", "id": "477fb1ed", "metadata": {}, "source": [ "## Bring edge table into Arrow format." ] }, { "cell_type": "code", "execution_count": 3, "id": "88bf9969", "metadata": { "scrolled": true }, "outputs": [ { "data": { "text/plain": [ "pyarrow.Table\n", "source: int64\n", "target: int64\n", "attr1: string\n", "attr2: string\n", "----\n", "source: [[6,6,9,2,4,...,7,3,8,9,10]]\n", "target: [[7,1,10,6,9,...,7,7,6,4,3]]\n", "attr1: [[\"x\",\"u\",\"u\",\"r\",\"k\",...,\"p\",\"x\",\"m\",\"q\",\"g\"]]\n", "attr2: [[\"a\",\"u\",\"j\",\"f\",\"d\",...,\"m\",\"g\",\"p\",\"g\",\"x\"]]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import pyarrow as pa\n", "import pyarrow.flight\n", "\n", "arrow_edges = pa.Table.from_pandas(df)\n", "arrow_edges" ] }, { "cell_type": "code", "execution_count": 4, "id": "aaa788e4", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['source', 'target', 'attr1', 'attr2']" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "arrow_edges.column_names" ] }, { "cell_type": "markdown", "id": "caefa9b0", "metadata": {}, "source": [ "## Send Arrow Flight to Rocketgraph xGT.\n", "\n", "First create frames on xGT to hold the graph data.\n", "\n", "Then establish a PyArrow Flight connection to the xGT server and send the data." ] }, { "cell_type": "code", "execution_count": 5, "id": "05bd7685", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import xgt\n", "server = xgt.Connection()\n", "if 'arrow ' not in server.get_namespaces():\n", " server.create_namespace('arrow')\n", "server.set_default_namespace('arrow')\n", "\n", "server.create_edge_frame_from_data(arrow_edges, 'Edge',\n", " source='Node', target='Node',\n", " source_key='source', target_key='target')" ] }, { "cell_type": "markdown", "id": "96f0a380", "metadata": {}, "source": [ "## Request to run a graph query on xGT." ] }, { "cell_type": "code", "execution_count": 6, "id": "b579443d", "metadata": { "scrolled": true }, "outputs": [], "source": [ "query = \"\"\"\n", "MATCH (a)-[e1:Edge]->(b)-[e2:Edge]->(c)\n", "WHERE a <> b and b <> c and a <> c\n", " AND e1.attr1 = e2.attr1\n", "RETURN DISTINCT a.id AS a, c.id AS c, e1.attr1 AS attr1\n", "INTO AnswerSet\n", "\"\"\"\n", "\n", "job = server.run_job(query)\n", "data = job.get_data(format='pandas')\n", "data" ] }, { "cell_type": "markdown", "id": "d3e99d2d", "metadata": {}, "source": [ "## Bring results back\n", "\n", "Note that this process uses the *Arrow Flight* mechanism to transfer the `AnswerSet` data from the xGT server to the client." ] }, { "cell_type": "code", "execution_count": 7, "id": "7db74d02", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "pyarrow.Table\n", "a: int64\n", "c: int64\n", "attr1: string\n", "----\n", "a: [[10,6,5,1,2,...,4,6,1,9,4]]\n", "c: [[7,3,3,10,3,...,3,5,6,4,6]]\n", "attr1: [[\"t\",\"q\",\"p\",\"u\",\"l\",...,\"q\",\"o\",\"r\",\"p\",\"q\"]]" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "answers = server.get_frame('AnswerSet').get_data(format='arrow')\n", "answers" ] }, { "cell_type": "markdown", "id": "3b321d3a", "metadata": {}, "source": [ "## Visualize results." ] }, { "cell_type": "code", "execution_count": 8, "id": "0fee52b8", "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "\n", "6\n", "\n", "6\n", "\n", "\n", "\n", "5\n", "\n", "5\n", "\n", "\n", "\n", "6->5\n", "\n", "\n", ""o"\n", "\n", "\n", "\n", "3\n", "\n", "3\n", "\n", "\n", "\n", "6->3\n", "\n", "\n", ""q"\n", "\n", "\n", "\n", "6->3\n", "\n", "\n", ""u"\n", "\n", "\n", "\n", "9\n", "\n", "9\n", "\n", "\n", "\n", "6->9\n", "\n", "\n", ""u"\n", "\n", "\n", "\n", "7\n", "\n", "7\n", "\n", "\n", "\n", "7->5\n", "\n", "\n", ""i"\n", "\n", "\n", "\n", "8\n", "\n", "8\n", "\n", "\n", "\n", "7->8\n", "\n", "\n", ""i"\n", "\n", "\n", "\n", "7->9\n", "\n", "\n", ""f"\n", "\n", "\n", "\n", "4\n", "\n", "4\n", "\n", "\n", "\n", "4->6\n", "\n", "\n", ""q"\n", "\n", "\n", "\n", "2\n", "\n", "2\n", "\n", "\n", "\n", "4->2\n", "\n", "\n", ""e"\n", "\n", "\n", "\n", "4->3\n", "\n", "\n", ""q"\n", "\n", "\n", "\n", "5->3\n", "\n", "\n", ""p"\n", "\n", "\n", "\n", "2->7\n", "\n", "\n", ""r"\n", "\n", "\n", "\n", "2->3\n", "\n", "\n", ""l"\n", "\n", "\n", "\n", "3->6\n", "\n", "\n", ""a"\n", "\n", "\n", "\n", "1\n", "\n", "1\n", "\n", "\n", "\n", "1->6\n", "\n", "\n", ""r"\n", "\n", "\n", "\n", "10\n", "\n", "10\n", "\n", "\n", "\n", "1->10\n", "\n", "\n", ""u"\n", "\n", "\n", "\n", "1->8\n", "\n", "\n", ""r"\n", "\n", "\n", "\n", "10->7\n", "\n", "\n", ""t"\n", "\n", "\n", "\n", "10->7\n", "\n", "\n", ""g"\n", "\n", "\n", "\n", "10->7\n", "\n", "\n", ""q"\n", "\n", "\n", "\n", "10->5\n", "\n", "\n", ""g"\n", "\n", "\n", "\n", "10->2\n", "\n", "\n", ""r"\n", "\n", "\n", "\n", "10->1\n", "\n", "\n", ""g"\n", "\n", "\n", "\n", "9->7\n", "\n", "\n", ""g"\n", "\n", "\n", "\n", "9->7\n", "\n", "\n", ""q"\n", "\n", "\n", "\n", "9->4\n", "\n", "\n", ""p"\n", "\n", "\n", "\n", "9->1\n", "\n", "\n", ""a"\n", "\n", "\n", "\n" ], "text/plain": [ "" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "#answers_table = pa.Table.from_pandas(answers)\n", "try:\n", " import graphviz\n", " from IPython.display import Image\n", " \n", " gr = graphviz.Digraph()\n", " # Draw nodes for the union of all source and target nodes in the two paths\n", " for node in set(answers.column('a')) | set(answers.column('c')):\n", " gr.node(str(node))\n", " # Draw edges for each returned edge\n", " for batch in answers.to_batches():\n", " d = batch.to_pydict()\n", " for a, c, attr1 in zip(d['a'], d['c'], d['attr1']):\n", " # Do something with the row\n", " gr.edge(str(a), str(c), label=f'\"{str(attr1)}\"')\n", "\n", "except ImportError:\n", " print(\"You must install graphviz in order to vizualize the results as a graph\")\n", " print(\"\\n!python3 -m pip install graphviz\")\n", " gr = None\n", "\n", "gr" ] }, { "cell_type": "markdown", "id": "4b7eb72c", "metadata": {}, "source": [ "" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.2" } }, "nbformat": 4, "nbformat_minor": 5 }