Building a BitClout Social Network Visualization App With Memgraph and D3.js

Learn how to develop a simple application for visualizing and analyzing the BitClout social network using Memgraph, Python, and use D3.js.

Image by author

Introduction

Prerequisites

Scraping the BitClout HODLers Data

Image by author

Creating The Flask Server Backend

import os
MG_HOST = os.getenv('MG_HOST', '127.0.0.1')
MG_PORT = int(os.getenv('MG_PORT', '7687'))
import logging
log = logging.getLogger(__name__)
def init_log():
logging.basicConfig(level=logging.INFO)
log.info("Logging enabled")
logging.getLogger("werkzeug").setLevel(logging.WARNING)
init_log()
from argparse import ArgumentParser
def parse_args():
parser = ArgumentParser(description=__doc__)
parser.add_argument("--app-host", default="0.0.0.0",
help="Host address.")
parser.add_argument("--app-port", default=5000, type=int,
help="App port.")
parser.add_argument("--template-folder", default="public/template",
help="Path to the directory with flask templates.")
parser.add_argument("--static-folder", default="public",
help="Path to the directory with flask static files.")
parser.add_argument("--debug", default=True, action="store_true",
help="Run web server in debug mode.")
parser.add_argument("--load-data", default=False, action='store_true',
help="Load BitClout network into Memgraph.")
print(__doc__)
return parser.parse_args()
args = parse_args()
import mgclient
import time
connection_established = False
while(not connection_established):
try:
connection = mgclient.connect(
host=MG_HOST,
port=MG_PORT,
username="",
password="",
sslmode=mgclient.MG_SSLMODE_DISABLE,
lazy=True)
connection_established = True
except:
log.info("Memgraph probably isn't running.")
time.sleep(4)
cursor = connection.cursor()
from flask import Flask
app = Flask(__name__,
template_folder=args.template_folder,
static_folder=args.static_folder,
static_url_path='')
import JSON
from flask import Response
@app.route('/load-all', methods=['GET'])
def load_all():
"""Load everything from the database."""
start_time = time.time()
try:
cursor.execute("""MATCH (n)-[r]-(m)
RETURN n, r, m
LIMIT 20000;""")
rows = cursor.fetchall()
except:
log.info("Something went wrong.")
return ('', 204)
links = []
nodes = []
visited = []
for row in rows:
n = row[0]
m = row[2]
if n.id not in visited:
nodes.append({'id': n.id})
visited.append(n.id)
if m.id not in visited:
nodes.append({'id': m.id})
visited.append(m.id)
links.append({'source': n.id, 'target': m.id})
response = {'nodes': nodes, 'links': links}
duration = time.time() - start_time
log.info("Data fetched in: " + str(duration) + " seconds")
return Response(
json.dumps(response),
status=200,
mimetype='application/json')
from flask import render_template
@app.route('/', methods=['GET'])
def index():
return render_template('index.html')
def main():
if args.load_data:
log.info("Loading the data into Memgraph.")
database.load_data(cursor)
app.run(host=args.app_host, port=args.app_port, debug=args.debug)
if __name__ == "__main__":
main()

Importing the BitClout Network into Memgraph

def load_data(cursor):
cursor.execute("""MATCH (n)
DETACH DELETE n;""")
cursor.fetchall()
cursor.execute("""CREATE INDEX ON :User(id);""")
cursor.fetchall()
cursor.execute("""CREATE INDEX ON :User(name);""")
cursor.fetchall()
cursor.execute("""CREATE CONSTRAINT ON (user:User)
ASSERT user.id IS UNIQUE;""")
cursor.fetchall()
cursor.execute("""LOAD CSV FROM '/usr/lib/memgraph/import-data/profiles-1.csv' 
WITH header AS row
CREATE (sample:User {id: row.id})
SET sample += {
name: row.name,
description: row.description,
image: row.image,
isHidden: row.isHidden,
isReserved: row.isReserved,
isVerified: row.isVerified,
coinPrice: row.coinPrice,
creatorBasisPoints: row.creatorBasisPoints,
lockedNanos: row.lockedNanos,
nanosInCirculation: row.nanosInCirculation,
watermarkNanos: row.watermarkNanos
};""")
cursor.fetchall()
cursor.execute("""LOAD CSV FROM '/usr/lib/memgraph/import-data/hodls.csv' 
WITH header AS row
MATCH (hodler:User {id: row.from})
MATCH (creator:User {id: row.to})
CREATE (hodler)-[:HODLS {amount: row.nanos}]->(creator);""")
cursor.fetchall()

Building Your Frontend with D3.js

<canvas width="800" height="600" style="border: ..."></canvas>
const width = 800;
const height = 600;
var links;
var nodes;
var simulation;
var transform;
var canvas = d3.select("canvas");
var context = canvas.node().getContext('2d');
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("GET", '/load-all', true);
xmlhttp.setRequestHeader('Content-type', 'application/json; charset=utf-8');
xmlhttp.onreadystatechange = function () {
if (xmlhttp.readyState == 4 && xmlhttp.status == "200") {
data = JSON.parse(xmlhttp.responseText);
links = data.links;
nodes = data.nodes;
simulation = d3.forceSimulation()
.force("center", d3.forceCenter(width / 2, height / 2))
.force("x", d3.forceX(width / 2).strength(0.1))
.force("y", d3.forceY(height / 2).strength(0.1))
.force("charge", d3.forceManyBody().strength(-50))
.force("link", d3.forceLink().strength(1).id(function (d) { return d.id; }))
.alphaTarget(0)
.alphaDecay(0.05);
transform = d3.zoomIdentity;
d3.select(context.canvas)
.call(d3.drag().subject(dragsubject).on("start", dragstarted).on("drag", dragged).on("end", dragended))
.call(d3.zoom().scaleExtent([1 / 10, 8]).on("zoom", zoomed));
simulation.nodes(nodes)
.on("tick", simulationUpdate);
simulation.force("link")
.links(links);
}
}
function simulationUpdate() {
context.save();
context.clearRect(0, 0, width, height);
context.translate(transform.x, transform.y);
context.scale(transform.k, transform.k);
links.forEach(function (d) {
context.beginPath();
context.moveTo(d.source.x, d.source.y);
context.lineTo(d.target.x, d.target.y);
context.stroke();
});
nodes.forEach(function (d, i) {
context.beginPath();
context.arc(d.x, d.y, radius, 0, 2 * Math.PI, true);
context.fillStyle = "#FFA500";
context.fill();
});

context.restore();
}
function dragstarted(event) {
if (!event.active) simulation.alphaTarget(0.3).restart();
event.subject.fx = transform.invertX(event.x);
event.subject.fy = transform.invertY(event.y);
}
function dragged(event) {
event.subject.fx = transform.invertX(event.x);
event.subject.fy = transform.invertY(event.y);
}
function dragended(event) {
if (!event.active) simulation.alphaTarget(0);
event.subject.fx = null;
event.subject.fy = null;
}

Setting up your Docker Environment

version: "3"
services:
memgraph:
image: "memgraph/memgraph:latest"
user: root
volumes:
- ./memgraph/entrypoint:/usr/lib/memgraph/entrypoint
- ./memgraph/import-data:/usr/lib/memgraph/import-data
- ./memgraph/mg_lib:/var/lib/memgraph
- ./memgraph/mg_log:/var/log/memgraph
- ./memgraph/mg_etc:/etc/memgraph
ports:
- "7687:7687"
bitclout:
build: .
volumes:
- .:/app
ports:
- "5000:5000"
environment:
MG_HOST: memgraph
MG_PORT: 7687
depends_on:
- memgraph
FROM python:3.8
# Install CMake
RUN apt-get update && \
apt-get --yes install cmake
# Install mgclient
RUN apt-get install -y git cmake make gcc g++ libssl-dev && \
git clone https://github.com/memgraph/mgclient.git /mgclient && \
cd mgclient && \
git checkout dd5dcaaed5d7c8b275fbfd5d2ecbfc5006fa5826 && \
mkdir build && \
cd build && \
cmake .. && \
make && \
make install
# Install packages
COPY requirements.txt ./
RUN pip3 install -r requirements.txt
# Copy the source code to the container
COPY public /app/public
COPY bitclout.py /app/bitclout.py
COPY database.py /app/database.py
WORKDIR /app
ENV FLASK_ENV=development
ENV LC_ALL=C.UTF-8
ENV LANG=C.UTF-8
ENTRYPOINT ["python3", "bitclout.py", "--load-data"]
Flask==1.1.2
pymgclient==1.0.0

Launching your Application

docker-compose build
docker-compose up
ENTRYPOINT ["python3", "bitclout.py"]

Conclusion

Developer Relations Engineer and Computer science graduate

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store