templating api

api/src/db/cache-client.ts

@@ -0,0 +1,94 @@
+import { REDIS_CONNECTION_URL } from "@/config"
+import { logger } from "@/utils/logger"
+import { RedisClientType, createClient } from "@redis/client"
+import { ScanOptions } from "@redis/client/dist/lib/commands/SCAN"
+
+class CacheClient {
+  protected client: RedisClientType
+  protected failures: number = 0
+
+  constructor() {
+    logger.info(`INIT CACHE: ${REDIS_CONNECTION_URL}`)
+    this.client = createClient({ url: REDIS_CONNECTION_URL })
+
+    this.client.on("error", (err) => {
+      this.onError(err)
+    })
+    this.client.on("connect", async () => {
+      this.failures = 0
+      await this.setValueNoExpire("TESTING", 123)
+      logger.info("Redis Client Connect")
+    })
+  }
+
+  // eslint-disable-next-line
+  onError(err: any) {
+    if (this.failures < 5) {
+      this.failures++
+      logger.error(`Redis Connection Error ${this.failures}: ${err.message}`)
+    } else if (this.failures == 5) {
+      this.failures++
+      logger.error("Giving up on Redis")
+    } else {
+      logger.error(`Redis Generic Error: ${err.message}`)
+    }
+  }
+
+  async getClient() {
+    if (!this.client.isOpen) await this.client.connect()
+    return this
+  }
+
+  // eslint-disable-next-line
+  async setValue(key: string, value: any, expireSeconds = 90) {
+    if (expireSeconds <= 0) this.setValueNoExpire(key, value)
+    else this.client.set(key, value, { EX: expireSeconds })
+  }
+  // eslint-disable-next-line
+  async setValueNoExpire(key: string, value: any) {
+    this.client.set(key, value)
+  }
+  async getValue(key: string) {
+    return this.client.get(key)
+  }
+  async deleteValue(key: string) {
+    return this.client.del(key)
+  }
+
+  async deleteValuesByPattern(pattern: string) {
+    const scanCommand = { MATCH: `${pattern}*` } as ScanOptions
+    let cursor = "0"
+
+    do {
+      const reply = await this.client.scan(cursor, scanCommand)
+      cursor = reply.cursor
+      const keys = reply.keys
+
+      if (keys.length > 0) {
+        await this.client.del(keys)
+      }
+    } while (cursor !== "0")
+  }
+
+  async getKeysByPattern(pattern: string): Promise<string[]> {
+    const scanCommand = { MATCH: `${pattern}*` } as ScanOptions
+    let cursor = "0"
+    let matches = new Array<string>()
+
+    do {
+      const reply = await this.client.scan(cursor, scanCommand)
+      cursor = reply.cursor
+      const keys = reply.keys
+
+      if (keys.length > 0) {
+        matches = matches.concat(keys)
+      }
+    } while (cursor !== "0")
+
+    return matches
+  }
+}
+
+const cache = new CacheClient()
+
+export default cache

api/src/db/data/README.md

@@ -0,0 +1,4 @@
+# DB Data
+
+This folder contains SQL scripts or JSON files that contain data for seeding the database.
+Most of these things will be converted into seeds and factories, but they live here in the meantime.

api/src/db/db-client.ts

@@ -0,0 +1,53 @@
+import { Sequelize, Options } from "@sequelize/core"
+import { PostgresDialect } from "@sequelize/postgres"
+import { isEmpty, isNil } from "lodash"
+
+import {
+  DB_DATABASE,
+  DB_HOST,
+  DB_PASSWORD,
+  DB_PORT,
+  DB_USERNAME,
+  NODE_ENV,
+  SEQUELIZE_LOGGING,
+} from "@/config"
+import compactSql from "@/utils/compact-sql"
+
+if (isEmpty(DB_DATABASE)) throw new Error("database name is unset.")
+if (isEmpty(DB_USERNAME)) throw new Error("database username is unset.")
+if (isEmpty(DB_PASSWORD)) throw new Error("database password is unset.")
+if (isEmpty(DB_HOST)) throw new Error("database host is unset.")
+if (isNil(DB_PORT) || isNaN(DB_PORT)) throw new Error("database port is unset.")
+
+function sqlLogger(query: string) {
+  console.log(compactSql(query))
+}
+
+// See https://sequelize.org/docs/v7/databases/postgres/
+export const SEQUELIZE_CONFIG: Options<PostgresDialect> = {
+  dialect: PostgresDialect,
+  database: DB_DATABASE,
+  user: DB_USERNAME,
+  password: DB_PASSWORD,
+  host: DB_HOST,
+  port: DB_PORT,
+  ssl: NODE_ENV !== "production" ? false : { rejectUnauthorized: false },
+  schema: "public", // default - explicit for clarity
+  logging: SEQUELIZE_LOGGING ? sqlLogger : false,
+  pool: {
+    max: 20,
+    min: 2,
+    acquire: 60_000,
+    idle: 10_000,
+    evict: 10_000,
+  },
+  define: {
+    underscored: true,
+    timestamps: true, // default - explicit for clarity.
+    paranoid: true, // adds deleted_at column
+  },
+}
+
+const db = new Sequelize(SEQUELIZE_CONFIG)
+
+export default db

api/src/db/db-migration-client.ts

@@ -0,0 +1,54 @@
+import path from "path"
+
+import knex, { Knex } from "knex"
+import { isEmpty, isNil, merge } from "lodash"
+
+import { DB_DATABASE, DB_HOST, DB_PASSWORD, DB_PORT, DB_USERNAME, NODE_ENV } from "@/config"
+
+if (isEmpty(DB_DATABASE)) throw new Error("database name is unset.")
+if (isEmpty(DB_USERNAME)) throw new Error("database username is unset.")
+if (isEmpty(DB_PASSWORD)) throw new Error("database password is unset.")
+if (isEmpty(DB_HOST)) throw new Error("database host is unset.")
+if (isNil(DB_PORT) || isNaN(DB_PORT)) throw new Error("database port is unset.")
+
+export function buildKnexConfig(options?: Knex.Config): Knex.Config {
+  return merge(
+    {
+      client: "pg",
+      connection: {
+        host: DB_HOST,
+        user: DB_USERNAME,
+        password: DB_PASSWORD,
+        database: DB_DATABASE,
+        port: DB_PORT,
+        ssl:
+          NODE_ENV !== "production"
+            ? false
+            : {
+                require: true, // Enforce SSL
+                rejectUnauthorized: false, // Disable certificate verification (common for Azure)
+              },
+        /* options: {
+          encrypt: true,
+          trustServerCertificate: DB_TRUST_SERVER_CERTIFICATE,
+        }, */
+      },
+      migrations: {
+        directory: path.resolve(__dirname, "./migrations"),
+        extension: "ts",
+        stub: path.resolve(__dirname, "./templates/sample-migration.ts"),
+      },
+      seeds: {
+        directory: path.resolve(__dirname, `./seeds/${NODE_ENV}`),
+        extension: "ts",
+        stub: path.resolve(__dirname, "./templates/sample-seed.ts"),
+      },
+    },
+    options
+  )
+}
+
+const config = buildKnexConfig()
+const dbMigrationClient = knex(config)
+
+export default dbMigrationClient

api/src/db/migrations/20260619204725_create-users-table.ts

@@ -0,0 +1,36 @@
+import type { Knex } from "knex"
+
+export async function up(knex: Knex): Promise<void> {
+  await knex.schema.createTable("users", function (table) {
+    table.increments("id").notNullable().primary()
+    table.string("email", 100).notNullable()
+    table.string("auth0_subject", 100).notNullable()
+    table.string("first_name", 100).notNullable()
+    table.string("last_name", 100).notNullable()
+    table.string("display_name", 200).notNullable()
+    table.string("roles", 255).notNullable()
+
+    table
+      .specificType("created_at", "TIMESTAMP WITH TIME ZONE")
+      .notNullable()
+      .defaultTo(knex.raw("CURRENT_TIMESTAMP(0)"))
+    table
+      .specificType("updated_at", "TIMESTAMP WITH TIME ZONE")
+      .notNullable()
+      .defaultTo(knex.raw("CURRENT_TIMESTAMP(0)"))
+    table.specificType("deleted_at", "TIMESTAMP WITH TIME ZONE")
+
+    table.unique(["email"], {
+      indexName: "users_email_unique",
+      predicate: knex.whereNull("deleted_at"),
+    })
+    table.unique(["auth0_subject"], {
+      indexName: "users_auth0_subject_unique",
+      predicate: knex.whereNull("deleted_at"),
+    })
+  })
+}
+
+export async function down(knex: Knex): Promise<void> {
+  await knex.schema.dropTable("users")
+}

api/src/db/migrator.ts

@@ -0,0 +1,74 @@
+import express, { Request, Response } from "express"
+import { join } from "path"
+
+import { NODE_ENV } from "@/config"
+import dbMigrationClient from "@/db/db-migration-client"
+import { logger } from "@/utils/logger"
+
+export class Migrator {
+  readonly migrationRouter
+
+  constructor() {
+    this.migrationRouter = express.Router()
+
+    this.migrationRouter.get("/", async (_req: Request, res: Response) => {
+      return res.json({ data: await this.listMigrations() })
+    })
+
+    this.migrationRouter.get("/up", async (_req: Request, res: Response) => {
+      try {
+        await this.migrateUp()
+      } catch (err) {
+        logger.error(err)
+      }
+      return res.json({ data: await migrator.listMigrations() })
+    })
+
+    this.migrationRouter.get("/down", async (_req: Request, res: Response) => {
+      try {
+        await this.migrateDown()
+      } catch (err) {
+        logger.error(err)
+      }
+      return res.json({ data: await this.listMigrations() })
+    })
+
+    this.migrationRouter.get("/seed/:environment", async (req: Request, res: Response) => {
+      try {
+        await this.seedUp(req.params.environment)
+      } catch (err) {
+        logger.error(err)
+      }
+      return res.json({ data: "Seeding" })
+    })
+  }
+
+  listMigrations() {
+    return dbMigrationClient.migrate.list({ directory: join(__dirname, "migrations") })
+  }
+
+  async migrateUp() {
+    logger.warn("-------- MIGRATE UP ---------")
+    return dbMigrationClient.migrate.up({ directory: join(__dirname, "migrations") })
+  }
+
+  async migrateDown() {
+    logger.warn("-------- MIGRATE DOWN ---------")
+    return dbMigrationClient.migrate.down({ directory: join(__dirname, "migrations") })
+  }
+
+  async migrateLatest() {
+    logger.warn("-------- MIGRATE LATEST ---------")
+    return dbMigrationClient.migrate.latest({ directory: join(__dirname, "migrations") })
+  }
+
+  async seedUp(environment?: string) {
+    logger.warn("-------- SEED UP ---------")
+    return dbMigrationClient.seed.run({
+      directory: join(__dirname, "seeds", environment || NODE_ENV),
+    })
+  }
+}
+const migrator = new Migrator()
+
+export default migrator

api/src/db/seeds/development/000.seed-system-user.ts

@@ -0,0 +1,31 @@
+import { CreationAttributes } from "@sequelize/core"
+import { isNil } from "lodash"
+
+import logger from "@/utils/logger"
+import { CreateService } from "@/services/users"
+import { User } from "@/models"
+
+export async function seed(): Promise<void> {
+  const systemUserAttributes: CreationAttributes<User> = {
+    email: "system.user@alphane.com",
+    auth0Subject: "NO_LOGIN_system.user@alphane.com",
+    firstName: "System",
+    lastName: "User",
+    displayName: "System User",
+    roles: [User.Roles.SYSTEM_ADMIN],
+  }
+
+  const user = await User.findOne({
+    where: {
+      email: systemUserAttributes.email,
+    },
+  })
+
+  if (isNil(user)) {
+    const createdUser = await CreateService.perform(systemUserAttributes)
+    logger.debug("System User created:", createdUser.dataValues)
+  } else {
+    await user.update(systemUserAttributes)
+    logger.debug("System User updated:", user.dataValues)
+  }
+}

api/src/db/seeds/production/000.seed-system-user.ts

@@ -0,0 +1,31 @@
+import { CreationAttributes } from "@sequelize/core"
+import { isNil } from "lodash"
+
+import logger from "@/utils/logger"
+import { CreateService } from "@/services/users"
+import { User } from "@/models"
+
+export async function seed(): Promise<void> {
+  const systemUserAttributes: CreationAttributes<User> = {
+    email: "system.user@alphane.com",
+    auth0Subject: "NO_LOGIN_system.user@alphane.com",
+    firstName: "System",
+    lastName: "User",
+    displayName: "System User",
+    roles: [User.Roles.SYSTEM_ADMIN],
+  }
+
+  const user = await User.findOne({
+    where: {
+      email: systemUserAttributes.email,
+    },
+  })
+
+  if (isNil(user)) {
+    const createdUser = await CreateService.perform(systemUserAttributes)
+    logger.debug("System User created:", createdUser.dataValues)
+  } else {
+    await user.update(systemUserAttributes)
+    logger.debug("System User updated:", user.dataValues)
+  }
+}

api/src/db/seeds/test/000.seed-system-user.ts

@@ -0,0 +1,31 @@
+import { CreationAttributes } from "@sequelize/core"
+import { isNil } from "lodash"
+
+import logger from "@/utils/logger"
+import { CreateService } from "@/services/users"
+import { User } from "@/models"
+
+export async function seed(): Promise<void> {
+  const systemUserAttributes: CreationAttributes<User> = {
+    email: "system.user@alphane.com",
+    auth0Subject: "NO_LOGIN_system.user@alphane.com",
+    firstName: "System",
+    lastName: "User",
+    displayName: "System User",
+    roles: [User.Roles.SYSTEM_ADMIN],
+  }
+
+  const user = await User.findOne({
+    where: {
+      email: systemUserAttributes.email,
+    },
+  })
+
+  if (isNil(user)) {
+    const createdUser = await CreateService.perform(systemUserAttributes)
+    logger.debug("System User created:", createdUser.dataValues)
+  } else {
+    await user.update(systemUserAttributes)
+    logger.debug("System User updated:", user.dataValues)
+  }
+}

api/src/db/templates/sample-migration.ts

@@ -0,0 +1,9 @@
+import type { Knex } from "knex"
+
+export async function up(knex: Knex): Promise<void> {
+  throw new Error("Not implemented")
+}
+
+export async function down(knex: Knex): Promise<void> {
+  throw new Error("Not implemented")
+}

api/src/db/templates/sample-seed.ts

@@ -0,0 +1,33 @@
+import { Knex } from "knex"
+import { isNil } from "lodash"
+
+import logger from "@/utils/logger"
+import { User } from "@/models"
+
+export async function seed(_knex: Knex): Promise<void> {
+  const usersAttributes = [
+    {
+      email: "system.user@richter-guardian.com",
+      auth0Subject: "system.user@richter-guardian.com",
+      firstName: "System",
+      lastName: "User",
+      displayName: "System User",
+      roles: [User.Roles.SYSTEM_ADMIN],
+      title: "System User",
+    },
+  ]
+  for (const attributes of usersAttributes) {
+    let user = await User.findOne({
+      where: {
+        email: attributes.email,
+      },
+    })
+    if (isNil(user)) {
+      user = await User.create(attributes)
+      logger.debug("User created:", user.dataValues)
+    } else {
+      await user.update(attributes)
+      logger.debug("User updated:", user.dataValues)
+    }
+  }
+}

api/src/db/test-connection.ts

@@ -0,0 +1,17 @@
+import dbMigrationClient from "@/db/db-migration-client"
+import { logger } from "@/utils/logger"
+
+export async function isValidConnection() {
+  try {
+    await dbMigrationClient.raw("SELECT GETDATE()")
+    logger.info("Connection has been established successfully.")
+    return true
+  } catch (error) {
+    logger.error("Unable to connect to the database: " + error)
+    return false
+  }
+}
+
+if (require.main === module) {
+  isValidConnection()
+}

api/src/db/utils/knex-query-to-sequelize-raw.ts

@@ -0,0 +1,54 @@
+import { Knex } from "knex"
+import { QueryTypes, QueryOptionsWithType } from "@sequelize/core"
+
+import db from "@/db/db-client"
+
+type QueryOptions = Omit<QueryOptionsWithType<QueryTypes.SELECT>, "bind" | "type">
+
+// TODO: fix types to show that it might return null
+export async function knexQueryToSequelizeSelect<T extends object>(
+  knexQuery: Knex.QueryBuilder,
+  options: QueryOptions = {}
+) {
+  const { sql: knexSql, bindings } = knexQuery.toSQL().toNative()
+  const { sql: sequelizeSql, bind } = knexSqlNativeToSequelizeQueryWithBind({
+    sql: knexSql,
+    bindings,
+  })
+  return db.query<T>(sequelizeSql, {
+    ...options,
+    bind,
+    type: QueryTypes.SELECT,
+  })
+}
+
+/**
+ * Note siganture is chosen so you can pass knexQuery.toSQL().toNative() directly
+ *
+ * Currently only tested with MSSQL dialect
+ *
+ * @param sqlWithKnexBindings knexQuery.toSQL().toNative().sql
+ * @param bindings knexQuery.toSQL().toNative().bindings
+ * @returns { sql: string, bind: unknown[] } in Sequelize format
+ */
+export function knexSqlNativeToSequelizeQueryWithBind({
+  sql: sqlWithKnexBindings,
+  bindings,
+}: {
+  sql: string
+  bindings: readonly unknown[]
+}): { sql: string; bind: unknown[] } {
+  let sqlWithSequelizeBindings = sqlWithKnexBindings
+  // converts "@p0" to "$1", "@p1" to "$2", etc.
+  bindings.forEach((_, i) => {
+    const pattern = new RegExp(`@p${i}\\b`, "g")
+    sqlWithSequelizeBindings = sqlWithSequelizeBindings.replace(pattern, `$${i + 1}`)
+  })
+
+  const mutableBindings = [...bindings]
+
+  return {
+    sql: sqlWithSequelizeBindings,
+    bind: mutableBindings,
+  }
+}

api/src/db/utils/safe-parse-json.ts

@@ -0,0 +1,18 @@
+import { logger } from "@/utils/logger"
+
+export function safeJsonParse(values: string): any[] {
+  try {
+    const lines = JSON.parse(values)
+    if (Array.isArray(lines)) {
+      return lines
+    } else {
+      logger.error("Parsed value is not an array.")
+      return []
+    }
+  } catch (error) {
+    logger.error(`Error parsing JSON: ${error}`, { error })
+    return []
+  }
+}
+
+export default safeJsonParse