Convert to SQLite+S3 backend rather than a dedicated PostgreSQL database

.gitignore

@@ -21,3 +21,5 @@ node_modules
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+chromium_downloads.db

README.md

@@ -1,4 +1,4 @@
-chromium-downloads
+# Chromium Downloads
 ==========
 
 [![Build Status](https://travis-ci.org/flotwig/chromium-downloads.svg?branch=master)](https://travis-ci.org/flotwig/chromium-downloads)
@@ -7,14 +7,75 @@ Live at: https://chromium.cypress.io
 
 ![image](https://user-images.githubusercontent.com/1151760/52878049-c29d0000-3129-11e9-8c71-e9497fc7a253.png)
 
+This application is a tool for tracking and downloading Chromium builds. It scrapes build data from the Chromium project and provides a user-friendly interface for viewing and downloading the builds. The application is split into a backend and a frontend, both written in JavaScript.
+
+The backend is an Express server that handles data scraping, storage, and retrieval. It uses SQLite for data storage and AWS S3 for database file synchronization. The frontend is a React application that provides a user interface for viewing and downloading the builds.
+
+## SQLite+S3
+
+In the recent changes, the application's database has been switched from PostgreSQL to SQLite. The SQLite database file is stored in the backend's local file system and is synchronized with an S3 bucket. This change was made to simplify the application's architecture and deployment.
+
+The SQLite database is managed by the sqlite3 library. The database file is named chromium_downloads.db and is located in the backend's root directory. The database schema is defined in backend/db.js and backend/s3.js.
+
+The S3 integration is handled by the `@aws-sdk/client-s3` and `@aws-sdk/lib-storage` libraries. The S3 bucket name is set by the `S3_BUCKET_NAME` environment variable. The S3 integration can be skipped by setting the `SKIP_S3_INTEGRATION` environment variable to true. This is useful for local development when you don't want to connect to S3.
+
+The database file is downloaded from S3 when the server starts and is uploaded to S3 when the server shuts down. This is handled in backend/index.js.
+
+The database file is also ignored by Git to prevent it from being committed to the repository.
+
 ### Installing dependencies
 
 ```
 yarn
 ```
 
-### Starting the dev server
+### Running
 
+
+To run the application, you need to install the dependencies with yarn and then start the server with yarn start. For local development, you can run the server without S3 integration by setting the SKIP_S3_INTEGRATION environment variable to true:
+
+```shell
+SKIP_S3_INTEGRATION=true yarn start
 ```
+
+The application uses several environment variables to configure its behavior:
+
+- `REACT_APP_API_URL`: This variable is used to set the API URL for the frontend. It is defined in frontend/src/index.js. If not set, it defaults to 'http://localhost:3001'.
+
+- `PORT`: This variable is used to set the port on which the backend server listens. It is defined in backend/index.js. If not set, it defaults to 3001.
+
+- `AWS_REGION`: This variable is used to set the AWS region for the S3 client. It is defined in backend/s3.js. If not set, it defaults to 'us-east-1'.
+
+- `S3_BUCKET_NAME`: This variable is used to set the name of the S3 bucket where the database file is stored. It is defined in backend/s3.js. There is no default value, so it must be set.
+
+- `SKIP_S3_INTEGRATION`: This variable is used to skip the integration with S3 for downloading and uploading the database file. It is defined in backend/s3.js and backend/index.js. If not set, it defaults to false.
+
+Please note that the `SKIP_S3_INTEGRATION` environment variable is used in the yarn start command above to run the application without S3 integration. This is useful for local development when you don't want to connect to S3.
+
+
+#### Running in Production
+
+To run the application in a production environment, you need to set the following environment variables:
+
+```shell
+REACT_APP_API_URL=<your_api_url>
+PORT=<your_port>
+AWS_REGION=<your_aws_region>
+S3_BUCKET_NAME=<your_s3_bucket_name>
+```
+
+Replace `<your_api_url>`, `<your_port>`, `<your_aws_region>`, and `<your_s3_bucket_name>` with your actual values.
+
+Then, you can start the server with:
+
+```shell
 yarn start
 ```
+
+Please note that in a production environment, you should not skip the S3 integration. The `SKIP_S3_INTEGRATION` environment variable should be left unset or set to false.
+
+The application leverages the [default AWS profile search paths](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-authentication.html#cli-chap-authentication-precedence), so an IAM role, profile, hard-coded credentials, etc will be respected.
+
+## Contributing
+
+Contributions are welcome. Please make sure to follow the coding style and add tests for any new features or changes.

backend/db.js

@@ -1,47 +1,24 @@
-const Sequelize = require("sequelize");
+const sqlite3 = require('sqlite3').verbose();
+const db = new sqlite3.Database('./chromium_downloads.db');
 
-let DATABASE_URL = process.env.DATABASE_URL;
+// Define the schema and create tables if they don't exist
+db.serialize(() => {
+  db.run(`CREATE TABLE IF NOT EXISTS builds (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    version TEXT,
+    os TEXT,
+    channel TEXT,
+    timestamp TEXT,
+    baseRevision TEXT,
+    artifactsRevision TEXT,
+    downloads TEXT
+  )`, (err) => {
+    if (err) {
+      console.error('Error creating builds table', err);
+    } else {
+      console.log('Successfully ensured the builds table exists');
+    }
+  });
+});
 
-if (process.env.NODE_ENV === "production") {
-  DATABASE_URL += "?ssl=true";
-}
-
-const sequelize = new Sequelize(DATABASE_URL);
-
-class Build extends Sequelize.Model {}
-Build.init(
-  {
-    version: Sequelize.STRING,
-    os: Sequelize.STRING,
-    channel: Sequelize.STRING,
-    timestamp: Sequelize.DATE,
-    baseRevision: Sequelize.STRING,
-    artifactsRevision: Sequelize.STRING,
-    downloads: Sequelize.JSONB,
-  },
-  {
-    sequelize,
-    modelName: "builds",
-    timestamps: false,
-    indexes: [
-      {
-        unique: true,
-        fields: ["version", "os", "channel", "timestamp"],
-      },
-    ],
-    pool: {
-      max: 4,
-      min: 1,
-    },
-  }
-);
-
-async function initialize() {
-  console.log(sequelize);
-  sequelize.sync();
-}
-
-module.exports = {
-  initialize,
-  Build,
-};
+module.exports = db;

backend/index.js

@@ -1,55 +1,77 @@
-const db = require("./db");
 const express = require("express");
+const db = require("./db");
 const scraper = require("./scraper");
+const { downloadDbFromS3, uploadDbToS3 } = require('./s3');
 
 const PORT = Number(process.env.PORT) || 3001;
-
 const app = express();
 
+// Middleware to allow cross-origin requests
 app.use((req, res, next) => {
   res.setHeader("access-control-allow-origin", "*");
-
   next();
 });
 
-app.get("/builds", (req, res) => {
-  db.Build.findAll({
-    attributes: ["version", "os", "channel", "timestamp"],
-    order: [["timestamp", "DESC"]],
-  }).then((builds) => {
+// Route definitions
+// Assuming `db` is the sqlite3 database object and is properly initialized
+
+app.get("/builds", async (req, res) => {
+  const query = "SELECT version, os, channel, timestamp FROM builds ORDER BY timestamp DESC";
+  db.all(query, [], (error, builds) => {
+    if (error) {
+      console.error(error);
+      return res.sendStatus(500);
+    }
     res.json(builds);
   });
 });
 
-app.get("/builds/:version/:channel/:os", (req, res) => {
-  db.Build.findAll({
-    where: {
-      channel: req.params.channel,
-      os: req.params.os,
-      version: req.params.version,
-    },
-  }).then((builds) => {
-    if (!builds.length) {
+app.get("/builds/:version/:channel/:os", async (req, res) => {
+  const { version, channel, os } = req.params;
+  const query = "SELECT * FROM builds WHERE channel = ? AND os = ? AND version = ?";
+  db.all(query, [channel, os, version], (error, builds) => {
+    if (error) {
+      console.error(error);
+      return res.sendStatus(500);
+    }
+    if (builds.length === 0) {
       return res.sendStatus(404);
     }
-
     res.json(builds[0]);
   });
 });
 
-console.log("Initializing");
-
-console.log(db.initialize);
-
-db.initialize()
-  .then(() => {
-    console.log("Starting scraping");
+// Server startup logic
+async function startServer() {
+  try {
+    await downloadDbFromS3();
+    console.log("Database downloaded from S3");
     scraper.start();
-
+    console.log("Scraper started");
     app.listen(PORT, () => {
       console.log(`Backend listening on ${PORT}.`);
     });
-  })
-  .catch((e) => {
-    console.error(e);
-  });
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+// Shutdown logic
+async function handleShutdown() {
+  try {
+    await uploadDbToS3();
+    console.log("Database uploaded to S3");
+    process.exit(0);
+  } catch (error) {
+    console.error(error);
+    process.exit(1);
+  }
+}
+
+// Handle shutdown signals
+process.on('SIGINT', handleShutdown);
+process.on('SIGTERM', handleShutdown);
+
+// Initialize the server
+startServer();

backend/package.json

@@ -3,12 +3,13 @@
   "version": "0.0.0",
   "private": true,
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.451.0",
+    "@aws-sdk/lib-storage": "^3.451.0",
     "bluebird": "^3.5.5",
     "express": "^4.17.1",
     "got": "^9.6.0",
     "nodemon": "^1.19.1",
-    "pg": "^8.8.0",
-    "sequelize": "^6.28.0"
+    "sqlite3": "^5.0.2"
   },
   "scripts": {
     "start": "node index.js",

backend/s3.js

@@ -0,0 +1,81 @@
+// backend/s3.js
+const { S3Client, GetObjectCommand } = require('@aws-sdk/client-s3');
+const { Upload } = require('@aws-sdk/lib-storage');
+const fs = require('fs');
+const sqlite3 = require('sqlite3').verbose();
+
+const s3Client = new S3Client({
+    region: process.env.AWS_REGION || 'us-east-1'
+  });
+
+const BUCKET_NAME = process.env.S3_BUCKET_NAME;
+const DB_FILE_NAME = 'chromium_downloads.db';
+const SKIP_S3_INTEGRATION = process.env.SKIP_S3_INTEGRATION === 'true';
+
+async function uploadDbToS3() {
+  if (SKIP_S3_INTEGRATION) {
+    return;
+  }
+
+  const fileStream = fs.createReadStream(DB_FILE_NAME);
+  const uploadParams = {
+    Bucket: BUCKET_NAME,
+    Key: DB_FILE_NAME,
+    Body: fileStream
+  };
+  const upload = new Upload({
+    client: s3Client,
+    params: uploadParams
+  });
+
+  await upload.done();
+}
+
+async function downloadDbFromS3() {
+  if (SKIP_S3_INTEGRATION) {
+    return;
+  }
+
+  const getObjectParams = {
+    Bucket: BUCKET_NAME,
+    Key: DB_FILE_NAME
+  };
+  try {
+    const { Body } = await s3Client.send(new GetObjectCommand(getObjectParams));
+    const fileStream = Body.pipe(fs.createWriteStream(DB_FILE_NAME));
+    await new Promise((resolve, reject) => {
+      fileStream.on('error', reject);
+      fileStream.on('close', resolve);
+    });
+  } catch (err) {
+    if (err.name === 'NoSuchKey') {
+      // Create an empty file
+      fs.writeFileSync(DB_FILE_NAME, '');
+      // Initialize the database and create the builds table
+      const db = new sqlite3.Database(DB_FILE_NAME);
+      db.serialize(() => {
+        db.run(`CREATE TABLE IF NOT EXISTS builds (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          version TEXT,
+          os TEXT,
+          channel TEXT,
+          timestamp TEXT,
+          baseRevision TEXT,
+          artifactsRevision TEXT,
+          downloads TEXT
+        )`, (err) => {
+          if (err) {
+            console.error('Error creating builds table', err);
+          } else {
+            console.log('Successfully created the builds table in new DB file');
+          }
+        });
+      });
+      db.close();
+    } else {
+      throw err;
+    }
+  }
+}
+
+module.exports = { uploadDbToS3, downloadDbFromS3 };