From fefbd45570fca99110fc500a30f8d7bb92c74e9b Mon Sep 17 00:00:00 2001 From: Patrick Fic Date: Wed, 14 Jan 2026 14:52:49 -0800 Subject: [PATCH] Separate infrastructure to separate yaml and base schema. --- serverless/README.md | 48 ++- serverless/drizzle/0000_fearless_vector.sql | 42 ++ serverless/drizzle/meta/0000_snapshot.json | 195 +++++++++ serverless/drizzle/meta/_journal.json | 13 + serverless/serverless.infra.yml | 397 ++++++++++++++++++ serverless/serverless.yml | 435 ++------------------ serverless/src/handlers/dbPing.ts | 77 +++- serverless/src/lib/db.ts | 91 ++-- 8 files changed, 847 insertions(+), 451 deletions(-) create mode 100644 serverless/drizzle/0000_fearless_vector.sql create mode 100644 serverless/drizzle/meta/0000_snapshot.json create mode 100644 serverless/drizzle/meta/_journal.json create mode 100644 serverless/serverless.infra.yml diff --git a/serverless/README.md b/serverless/README.md index 6cf0e40..1bf7cd5 100644 --- a/serverless/README.md +++ b/serverless/README.md @@ -46,36 +46,51 @@ src/ The project uses `serverless-esbuild` for bundling TypeScript code for deployment. ```bash -# Deploy to dev stage -sls deploy --stage dev +# 1) Deploy shared infra (VPC + DB + Proxy + Secret) once +sls deploy --config serverless.infra.yml --stage shared -# Deploy to production +# 2) Deploy the application Lambdas per stage (dev/alpha/beta/prod) +sls deploy --stage dev sls deploy --stage prod ``` ## RDS + Drizzle (Postgres) -This project provisions a **private** Postgres RDS instance and the networking needed for Lambdas to reach it. +This project provisions a **shared private** Postgres RDS instance (one DB for all app stages) and the networking needed for Lambdas to reach it. + +If you want direct access from your laptop, the infra stack also allows inbound Postgres from a single admin IP CIDR (default is the current value in `serverless.infra.yml`). ### What gets created -Defined in `serverless.yml`: +Defined in `serverless.infra.yml`: - A dedicated VPC with 2 public subnets + 2 private subnets - A NAT Gateway for private-subnet egress - Security groups: - Lambda SG (egress all) - - RDS SG (allows inbound `5432` only from the Lambda SG) + - RDS Proxy SG (allows inbound `5432` only from the Lambda SG) + - RDS SG (allows inbound `5432` only from the Proxy SG) - An RDS Postgres instance: - `DeletionProtection: true` - `DeletionPolicy: Snapshot` / `UpdateReplacePolicy: Snapshot` - `AutoMinorVersionUpgrade: true` (minor updates) - - Not publicly accessible + - Publicly accessible (restricted to a single `admin_cidr`) - An RDS Proxy in the same VPC (Lambdas connect to the proxy, proxy connects to RDS) - A Secrets Manager secret for the RDS master user (generated password) Note: RDS + NAT Gateway incur AWS costs. +### Direct DB access (your IP only) + +Infra config includes an `admin_cidr` parameter (e.g. `70.36.57.88/32`) that is allowed to connect to port `5432`. The RDS instance is placed in public subnets so it can be reached directly, but security groups restrict it to that single CIDR. + +To change it, edit `stages.shared.params.admin_cidr` in `serverless.infra.yml` and redeploy infra: + +```bash +cd serverless +sls deploy --config serverless.infra.yml --stage shared +``` + ### Drizzle files - Schema: `src/db/schema/**/*.ts` @@ -104,7 +119,7 @@ This writes SQL into `serverless/drizzle/`. ```bash cd serverless -sls deploy --stage dev +sls deploy --config serverless.infra.yml --stage shared ``` RDS creation can take several minutes. @@ -137,12 +152,12 @@ npm run db:migrate ### Deletion protection behavior -- `sls remove` will **not** be able to delete the DB instance while deletion protection is enabled. -- To intentionally remove it later, you must first disable deletion protection in `serverless.yml` and redeploy, then remove the stack. +- `sls remove --config serverless.infra.yml --stage shared` will **not** be able to delete the DB instance while deletion protection is enabled. +- To intentionally remove it later, you must first disable deletion protection in `serverless.infra.yml` and redeploy, then remove the infra stack. -### Stage-specific DB name +### Single shared DB -The Postgres database name comes from the stage param `db_name` in `serverless.yml` (e.g. `esdpdev`, `esdpalpha`, `esdpbeta`, `esdpprod`). +The DB name comes from the `shared` stage param `db_name` in `serverless.infra.yml` (defaults to `esdp`). All app stages connect to this same DB via CloudFormation outputs. ## Local Development @@ -156,7 +171,8 @@ sls dev - **TypeScript**: Configured in `tsconfig.json` - **ESLint**: Configured in `eslint.config.mjs` (ESM flat config) - **Prettier**: Configured in `.prettierrc.json` -- **Serverless**: Configured in `serverless.yml` +- **Serverless (app)**: Configured in `serverless.yml` +- **Serverless (infra)**: Configured in `serverless.infra.yml` ## Code Quality @@ -169,3 +185,9 @@ This project enforces: ## Environment Variables See `serverless.yml` for environment-specific configuration. + + +How To Deploy + +Deploy infra once (shared DB + networking): cd serverless && sls deploy --config serverless.infra.yml --stage shared +Deploy/update Lambdas per environment without touching DB/VPC: cd serverless && sls deploy --stage dev (and similarly --stage prod, etc.) \ No newline at end of file diff --git a/serverless/drizzle/0000_fearless_vector.sql b/serverless/drizzle/0000_fearless_vector.sql new file mode 100644 index 0000000..222e849 --- /dev/null +++ b/serverless/drizzle/0000_fearless_vector.sql @@ -0,0 +1,42 @@ +CREATE TABLE + "joblines" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid () NOT NULL, + "created_at" timestamp + with + time zone DEFAULT now () NOT NULL, + "jobId" uuid NOT NULL, + "line_desc" text + ); + +--> statement-breakpoint +CREATE TABLE + "jobs" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid () NOT NULL, + "shopId" uuid NOT NULL, + "created_at" timestamp + with + time zone DEFAULT now () NOT NULL, + "clm_no" text, + "ciecaid" text + ); + +--> statement-breakpoint +CREATE TABLE + "shops" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid () NOT NULL, + "created_at" timestamp + with + time zone DEFAULT now () NOT NULL, + "es_api_key" text NOT NULL, + "active" boolean DEFAULT true NOT NULL, + CONSTRAINT "shops_es_api_key_unique" UNIQUE ("es_api_key") + ); + +--> statement-breakpoint +ALTER TABLE "joblines" ADD CONSTRAINT "joblines_jobId_jobs_id_fk" FOREIGN KEY ("jobId") REFERENCES "public"."jobs" ("id") ON DELETE no action ON UPDATE no action; + +--> statement-breakpoint +ALTER TABLE "jobs" ADD CONSTRAINT "jobs_shopId_shops_id_fk" FOREIGN KEY ("shopId") REFERENCES "public"."shops" ("id") ON DELETE no action ON UPDATE no action; + +--> statement-breakpoint +CREATE INDEX "clm_no_idx" ON "jobs" USING btree ("clm_no"); \ No newline at end of file diff --git a/serverless/drizzle/meta/0000_snapshot.json b/serverless/drizzle/meta/0000_snapshot.json new file mode 100644 index 0000000..066efe7 --- /dev/null +++ b/serverless/drizzle/meta/0000_snapshot.json @@ -0,0 +1,195 @@ +{ + "id": "e789ff38-2370-43ff-825a-fde4c3efca40", + "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.joblines": { + "name": "joblines", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "jobId": { + "name": "jobId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "line_desc": { + "name": "line_desc", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "joblines_jobId_jobs_id_fk": { + "name": "joblines_jobId_jobs_id_fk", + "tableFrom": "joblines", + "tableTo": "jobs", + "columnsFrom": [ + "jobId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.jobs": { + "name": "jobs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "shopId": { + "name": "shopId", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "clm_no": { + "name": "clm_no", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ciecaid": { + "name": "ciecaid", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "clm_no_idx": { + "name": "clm_no_idx", + "columns": [ + { + "expression": "clm_no", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "jobs_shopId_shops_id_fk": { + "name": "jobs_shopId_shops_id_fk", + "tableFrom": "jobs", + "tableTo": "shops", + "columnsFrom": [ + "shopId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.shops": { + "name": "shops", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "es_api_key": { + "name": "es_api_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "shops_es_api_key_unique": { + "name": "shops_es_api_key_unique", + "nullsNotDistinct": false, + "columns": [ + "es_api_key" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/serverless/drizzle/meta/_journal.json b/serverless/drizzle/meta/_journal.json new file mode 100644 index 0000000..7a2c104 --- /dev/null +++ b/serverless/drizzle/meta/_journal.json @@ -0,0 +1,13 @@ +{ + "version": "7", + "dialect": "postgresql", + "entries": [ + { + "idx": 0, + "version": "7", + "when": 1768419700617, + "tag": "0000_fearless_vector", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/serverless/serverless.infra.yml b/serverless/serverless.infra.yml new file mode 100644 index 0000000..ffa9f51 --- /dev/null +++ b/serverless/serverless.infra.yml @@ -0,0 +1,397 @@ +service: esdp-infra +app: esdp-api-app +frameworkVersion: '4' + +stages: + shared: + observability: false + params: + # Single shared database name for all app stages (dev/alpha/beta/prod) + db_name: esdp + # Your public IP in CIDR form for direct DB access (lock this down). + admin_cidr: 70.36.57.88/32 + +provider: + name: aws + runtime: nodejs22.x + region: ca-central-1 + +resources: + Resources: + EsdpVpc: + Type: AWS::EC2::VPC + Properties: + CidrBlock: 10.0.0.0/16 + EnableDnsSupport: true + EnableDnsHostnames: true + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-vpc + + InternetGateway: + Type: AWS::EC2::InternetGateway + Properties: + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-igw + + VpcGatewayAttachment: + Type: AWS::EC2::VPCGatewayAttachment + Properties: + VpcId: + Ref: EsdpVpc + InternetGatewayId: + Ref: InternetGateway + + PublicSubnetA: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: EsdpVpc + AvailabilityZone: + Fn::Select: + - 0 + - Fn::GetAZs: '' + CidrBlock: 10.0.0.0/24 + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-public-a + + PublicSubnetB: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: EsdpVpc + AvailabilityZone: + Fn::Select: + - 1 + - Fn::GetAZs: '' + CidrBlock: 10.0.1.0/24 + MapPublicIpOnLaunch: true + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-public-b + + PrivateSubnetA: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: EsdpVpc + AvailabilityZone: + Fn::Select: + - 0 + - Fn::GetAZs: '' + CidrBlock: 10.0.10.0/24 + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-private-a + + PrivateSubnetB: + Type: AWS::EC2::Subnet + Properties: + VpcId: + Ref: EsdpVpc + AvailabilityZone: + Fn::Select: + - 1 + - Fn::GetAZs: '' + CidrBlock: 10.0.11.0/24 + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-private-b + + PublicRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: EsdpVpc + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-public-rt + + PublicRoute: + Type: AWS::EC2::Route + DependsOn: VpcGatewayAttachment + Properties: + RouteTableId: + Ref: PublicRouteTable + DestinationCidrBlock: 0.0.0.0/0 + GatewayId: + Ref: InternetGateway + + PublicSubnetARouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: + Ref: PublicSubnetA + RouteTableId: + Ref: PublicRouteTable + + PublicSubnetBRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: + Ref: PublicSubnetB + RouteTableId: + Ref: PublicRouteTable + + NatEip: + Type: AWS::EC2::EIP + Properties: + Domain: vpc + + NatGateway: + Type: AWS::EC2::NatGateway + Properties: + AllocationId: + Fn::GetAtt: + - NatEip + - AllocationId + SubnetId: + Ref: PublicSubnetA + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-nat + + PrivateRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: + Ref: EsdpVpc + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-private-rt + + PrivateRoute: + Type: AWS::EC2::Route + Properties: + RouteTableId: + Ref: PrivateRouteTable + DestinationCidrBlock: 0.0.0.0/0 + NatGatewayId: + Ref: NatGateway + + PrivateSubnetARouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: + Ref: PrivateSubnetA + RouteTableId: + Ref: PrivateRouteTable + + PrivateSubnetBRouteTableAssociation: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + SubnetId: + Ref: PrivateSubnetB + RouteTableId: + Ref: PrivateRouteTable + + LambdaSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: ${self:service}-${sls:stage} Lambda security group + VpcId: + Ref: EsdpVpc + SecurityGroupEgress: + - IpProtocol: -1 + CidrIp: 0.0.0.0/0 + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-lambda-sg + + ProxySecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: ${self:service}-${sls:stage} RDS proxy security group + VpcId: + Ref: EsdpVpc + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + SourceSecurityGroupId: + Ref: LambdaSecurityGroup + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + CidrIp: ${param:admin_cidr} + SecurityGroupEgress: + - IpProtocol: -1 + CidrIp: 0.0.0.0/0 + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-proxy-sg + + RdsSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: ${self:service}-${sls:stage} RDS security group + VpcId: + Ref: EsdpVpc + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + SourceSecurityGroupId: + Ref: ProxySecurityGroup + - IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + CidrIp: ${param:admin_cidr} + Tags: + - Key: Name + Value: ${self:service}-${sls:stage}-rds-sg + + DbSubnetGroup: + Type: AWS::RDS::DBSubnetGroup + Properties: + DBSubnetGroupDescription: ${self:service}-${sls:stage} DB subnet group + SubnetIds: + - Ref: PublicSubnetA + - Ref: PublicSubnetB + + DbSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: ${self:service}-${sls:stage} RDS master credentials + GenerateSecretString: + SecretStringTemplate: '{"username":"esdp_admin"}' + GenerateStringKey: password + PasswordLength: 32 + ExcludeCharacters: '"@/\\' + + DbProxyRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - rds.amazonaws.com + Action: + - sts:AssumeRole + Policies: + - PolicyName: ${self:service}-${sls:stage}-db-proxy-secrets + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - Ref: DbSecret + + DbProxy: + Type: AWS::RDS::DBProxy + Properties: + DBProxyName: ${self:service}-${sls:stage}-proxy + EngineFamily: POSTGRESQL + IdleClientTimeout: 1800 + RequireTLS: true + RoleArn: + Fn::GetAtt: + - DbProxyRole + - Arn + VpcSubnetIds: + - Ref: PrivateSubnetA + - Ref: PrivateSubnetB + VpcSecurityGroupIds: + - Ref: ProxySecurityGroup + Auth: + - AuthScheme: SECRETS + SecretArn: + Ref: DbSecret + IAMAuth: DISABLED + + DbProxyTargetGroup: + Type: AWS::RDS::DBProxyTargetGroup + Properties: + DBProxyName: + Ref: DbProxy + TargetGroupName: default + DBInstanceIdentifiers: + - Ref: PostgresDb + ConnectionPoolConfigurationInfo: + MaxConnectionsPercent: 75 + MaxIdleConnectionsPercent: 50 + ConnectionBorrowTimeout: 120 + + PostgresDb: + Type: AWS::RDS::DBInstance + DeletionPolicy: Snapshot + UpdateReplacePolicy: Snapshot + Properties: + DBInstanceIdentifier: ${self:service}-${sls:stage}-postgres + Engine: postgres + # EngineVersion intentionally omitted so AWS uses the current default/latest for RDS Postgres. + AutoMinorVersionUpgrade: true + DBInstanceClass: db.t4g.micro + AllocatedStorage: 20 + StorageType: gp3 + StorageEncrypted: true + PubliclyAccessible: true + MultiAZ: false + DBName: ${param:db_name} + BackupRetentionPeriod: 7 + CopyTagsToSnapshot: true + DeletionProtection: true + VPCSecurityGroups: + - Ref: RdsSecurityGroup + DBSubnetGroupName: + Ref: DbSubnetGroup + MasterUsername: + Fn::Sub: '{{resolve:secretsmanager:${DbSecret}::username}}' + MasterUserPassword: + Fn::Sub: '{{resolve:secretsmanager:${DbSecret}::password}}' + + Outputs: + VpcId: + Value: + Ref: EsdpVpc + PrivateSubnetAId: + Value: + Ref: PrivateSubnetA + PrivateSubnetBId: + Value: + Ref: PrivateSubnetB + + LambdaSecurityGroupId: + Value: + Ref: LambdaSecurityGroup + ProxySecurityGroupId: + Value: + Ref: ProxySecurityGroup + RdsSecurityGroupId: + Value: + Ref: RdsSecurityGroup + + DbName: + Value: ${param:db_name} + DbPort: + Value: 5432 + + DbEndpointAddress: + Value: + Fn::GetAtt: + - PostgresDb + - Endpoint.Address + DbEndpointPort: + Value: + Fn::GetAtt: + - PostgresDb + - Endpoint.Port + DbSecretArn: + Value: + Ref: DbSecret + + DbProxyEndpoint: + Value: + Fn::GetAtt: + - DbProxy + - Endpoint diff --git a/serverless/serverless.yml b/serverless/serverless.yml index ac4331a..edc0e23 100644 --- a/serverless/serverless.yml +++ b/serverless/serverless.yml @@ -17,7 +17,8 @@ stages: domain: es.imex.online es_user: Imex2 es_password: Patrick - db_name: esdpprod + infra_service: esdp-infra + infra_stage: shared beta: # Enables observability in the prod stage observability: false @@ -28,7 +29,8 @@ stages: domain: beta.es.imex.online es_user: Imex2 es_password: Patrick - db_name: esdpbeta + infra_service: esdp-infra + infra_stage: shared alpha: # Enables observability in the prod stage observability: false @@ -38,7 +40,8 @@ stages: domain: alpha.es.imex.online es_user: Imex2 es_password: Patrick - db_name: esdpalpha + infra_service: esdp-infra + infra_stage: shared dev: # Enables observability in the prod stage observability: false @@ -48,7 +51,17 @@ stages: domain: dev.es.imex.online es_user: Imex2 es_password: Patrick - db_name: esdpdev + infra_service: esdp-infra + infra_stage: shared + +custom: + infra_stack: ${param:infra_service}-${param:infra_stage} + + db: + host: ${cf:${self:custom.infra_stack}.DbProxyEndpoint} + port: ${cf:${self:custom.infra_stack}.DbPort} + name: ${cf:${self:custom.infra_stack}.DbName} + secretArn: ${cf:${self:custom.infra_stack}.DbSecretArn} # params: # dev: @@ -68,6 +81,31 @@ provider: httpApi: # This creates a cheaper, faster "HTTP API" Gateway cors: true # Automatically configures CORS + # Ensure all Lambdas can reach the shared RDS Proxy in the infra VPC + vpc: + securityGroupIds: + - ${cf:${self:custom.infra_stack}.LambdaSecurityGroupId} + subnetIds: + - ${cf:${self:custom.infra_stack}.PrivateSubnetAId} + - ${cf:${self:custom.infra_stack}.PrivateSubnetBId} + + # Default DB connection settings for all Lambdas (used by src/lib/db.ts) + environment: + DB_HOST: ${self:custom.db.host} + DB_PORT: ${self:custom.db.port} + DB_NAME: ${self:custom.db.name} + DB_SECRET_ARN: ${self:custom.db.secretArn} + + # Allow Lambdas to fetch the DB credentials from Secrets Manager + iam: + role: + statements: + - Effect: Allow + Action: + - secretsmanager:GetSecretValue + Resource: + - ${self:custom.db.secretArn} + build: esbuild: bundle: true @@ -117,390 +155,26 @@ functions: handler: src/handlers/dbMigrate.handler timeout: 30 memorySize: 512 - vpc: - securityGroupIds: - - Ref: LambdaSecurityGroup - subnetIds: - - Ref: PrivateSubnetA - - Ref: PrivateSubnetB - environment: - DB_HOST: - Fn::GetAtt: - - DbProxy - - Endpoint - DB_PORT: 5432 - DB_NAME: ${param:db_name} - DB_SECRET_ARN: - Ref: DbSecret iamRoleStatements: - Effect: Allow Action: - secretsmanager:GetSecretValue Resource: - - Ref: DbSecret + - ${cf:${self:custom.infra_stack}.DbSecretArn} dbPing: handler: src/handlers/dbPing.handler timeout: 15 memorySize: 256 - vpc: - securityGroupIds: - - Ref: LambdaSecurityGroup - subnetIds: - - Ref: PrivateSubnetA - - Ref: PrivateSubnetB - environment: - DB_HOST: - Fn::GetAtt: - - DbProxy - - Endpoint - DB_PORT: 5432 - DB_NAME: ${param:db_name} - DB_SECRET_ARN: - Ref: DbSecret iamRoleStatements: - Effect: Allow Action: - secretsmanager:GetSecretValue Resource: - - Ref: DbSecret + - ${cf:${self:custom.infra_stack}.DbSecretArn} resources: Resources: - EsdpVpc: - Type: AWS::EC2::VPC - Properties: - CidrBlock: 10.0.0.0/16 - EnableDnsSupport: true - EnableDnsHostnames: true - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-vpc - - InternetGateway: - Type: AWS::EC2::InternetGateway - Properties: - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-igw - - VpcGatewayAttachment: - Type: AWS::EC2::VPCGatewayAttachment - Properties: - VpcId: - Ref: EsdpVpc - InternetGatewayId: - Ref: InternetGateway - - PublicSubnetA: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: EsdpVpc - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: '' - CidrBlock: 10.0.0.0/24 - MapPublicIpOnLaunch: true - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-public-a - - PublicSubnetB: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: EsdpVpc - AvailabilityZone: - Fn::Select: - - 1 - - Fn::GetAZs: '' - CidrBlock: 10.0.1.0/24 - MapPublicIpOnLaunch: true - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-public-b - - PrivateSubnetA: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: EsdpVpc - AvailabilityZone: - Fn::Select: - - 0 - - Fn::GetAZs: '' - CidrBlock: 10.0.10.0/24 - MapPublicIpOnLaunch: false - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-private-a - - PrivateSubnetB: - Type: AWS::EC2::Subnet - Properties: - VpcId: - Ref: EsdpVpc - AvailabilityZone: - Fn::Select: - - 1 - - Fn::GetAZs: '' - CidrBlock: 10.0.11.0/24 - MapPublicIpOnLaunch: false - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-private-b - - PublicRouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: - Ref: EsdpVpc - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-public-rt - - PublicRoute: - Type: AWS::EC2::Route - DependsOn: VpcGatewayAttachment - Properties: - RouteTableId: - Ref: PublicRouteTable - DestinationCidrBlock: 0.0.0.0/0 - GatewayId: - Ref: InternetGateway - - PublicSubnetARouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: - Ref: PublicSubnetA - RouteTableId: - Ref: PublicRouteTable - - PublicSubnetBRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: - Ref: PublicSubnetB - RouteTableId: - Ref: PublicRouteTable - - NatEip: - Type: AWS::EC2::EIP - Properties: - Domain: vpc - - NatGateway: - Type: AWS::EC2::NatGateway - Properties: - AllocationId: - Fn::GetAtt: - - NatEip - - AllocationId - SubnetId: - Ref: PublicSubnetA - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-nat - - PrivateRouteTable: - Type: AWS::EC2::RouteTable - Properties: - VpcId: - Ref: EsdpVpc - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-private-rt - - PrivateRoute: - Type: AWS::EC2::Route - Properties: - RouteTableId: - Ref: PrivateRouteTable - DestinationCidrBlock: 0.0.0.0/0 - NatGatewayId: - Ref: NatGateway - - PrivateSubnetARouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: - Ref: PrivateSubnetA - RouteTableId: - Ref: PrivateRouteTable - - PrivateSubnetBRouteTableAssociation: - Type: AWS::EC2::SubnetRouteTableAssociation - Properties: - SubnetId: - Ref: PrivateSubnetB - RouteTableId: - Ref: PrivateRouteTable - - LambdaSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: ${self:service}-${sls:stage} Lambda security group - VpcId: - Ref: EsdpVpc - SecurityGroupEgress: - - IpProtocol: -1 - CidrIp: 0.0.0.0/0 - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-lambda-sg - - ProxySecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: ${self:service}-${sls:stage} RDS proxy security group - VpcId: - Ref: EsdpVpc - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - SourceSecurityGroupId: - Ref: LambdaSecurityGroup - - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - CidrIp: 70.36.57.88/32 - SecurityGroupEgress: - - IpProtocol: -1 - CidrIp: 0.0.0.0/0 - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-proxy-sg - - RdsSecurityGroup: - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: ${self:service}-${sls:stage} RDS security group - VpcId: - Ref: EsdpVpc - SecurityGroupIngress: - - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - SourceSecurityGroupId: - Ref: ProxySecurityGroup - - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - CidrIp: 70.36.57.88/32 - Tags: - - Key: Name - Value: ${self:service}-${sls:stage}-rds-sg - - DbSubnetGroup: - Type: AWS::RDS::DBSubnetGroup - Properties: - DBSubnetGroupDescription: ${self:service}-${sls:stage} DB subnet group - SubnetIds: - - Ref: PrivateSubnetA - - Ref: PrivateSubnetB - - Ref: PublicSubnetA - - Ref: PublicSubnetB - - DbSecret: - Type: AWS::SecretsManager::Secret - Properties: - Description: ${self:service}-${sls:stage} RDS master credentials - GenerateSecretString: - SecretStringTemplate: '{"username":"esdp_admin"}' - GenerateStringKey: password - PasswordLength: 32 - ExcludeCharacters: '"@/\\' - - DbProxyRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - rds.amazonaws.com - Action: - - sts:AssumeRole - Policies: - - PolicyName: ${self:service}-${sls:stage}-db-proxy-secrets - PolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Action: - - secretsmanager:GetSecretValue - Resource: - - Ref: DbSecret - - DbProxy: - Type: AWS::RDS::DBProxy - Properties: - DBProxyName: ${self:service}-${sls:stage}-proxy - EngineFamily: POSTGRESQL - IdleClientTimeout: 1800 - RequireTLS: true - RoleArn: - Fn::GetAtt: - - DbProxyRole - - Arn - VpcSubnetIds: - - Ref: PrivateSubnetA - - Ref: PrivateSubnetB - VpcSecurityGroupIds: - - Ref: ProxySecurityGroup - Auth: - - AuthScheme: SECRETS - SecretArn: - Ref: DbSecret - IAMAuth: DISABLED - - DbProxyTargetGroup: - Type: AWS::RDS::DBProxyTargetGroup - Properties: - DBProxyName: - Ref: DbProxy - TargetGroupName: default - DBInstanceIdentifiers: - - Ref: PostgresDb - ConnectionPoolConfigurationInfo: - MaxConnectionsPercent: 75 - MaxIdleConnectionsPercent: 50 - ConnectionBorrowTimeout: 120 - - PostgresDb: - Type: AWS::RDS::DBInstance - DeletionPolicy: Snapshot - UpdateReplacePolicy: Snapshot - Properties: - DBInstanceIdentifier: ${self:service}-${sls:stage}-postgres - Engine: postgres - # EngineVersion intentionally omitted so AWS uses the current default/latest for RDS Postgres. - AutoMinorVersionUpgrade: true - DBInstanceClass: db.t4g.micro - AllocatedStorage: 20 - StorageType: gp3 - StorageEncrypted: true - PubliclyAccessible: true - MultiAZ: false - DBName: ${param:db_name} - BackupRetentionPeriod: 7 - CopyTagsToSnapshot: true - DeletionProtection: true - VPCSecurityGroups: - - Ref: RdsSecurityGroup - DBSubnetGroupName: - Ref: DbSubnetGroup - MasterUsername: - Fn::Sub: '{{resolve:secretsmanager:${DbSecret}::username}}' - MasterUserPassword: - Fn::Sub: '{{resolve:secretsmanager:${DbSecret}::password}}' - UploadBucket: Type: AWS::S3::Bucket Properties: @@ -514,24 +188,3 @@ resources: - POST AllowedHeaders: - '*' - - Outputs: - DbEndpointAddress: - Value: - Fn::GetAtt: - - PostgresDb - - Endpoint.Address - DbEndpointPort: - Value: - Fn::GetAtt: - - PostgresDb - - Endpoint.Port - DbSecretArn: - Value: - Ref: DbSecret - - DbProxyEndpoint: - Value: - Fn::GetAtt: - - DbProxy - - Endpoint diff --git a/serverless/src/handlers/dbPing.ts b/serverless/src/handlers/dbPing.ts index 399018e..5af6a24 100644 --- a/serverless/src/handlers/dbPing.ts +++ b/serverless/src/handlers/dbPing.ts @@ -7,17 +7,88 @@ import { getDb } from '../lib/db'; export const handler = async (): Promise => { try { const db = await getDb(); - const result = await db.execute(sql`select 1 as ok`); + const ping = await db.execute(sql`select 1 as ok`); + + const schemaRows = await db.execute(sql` + select + table_schema, + table_name, + column_name, + data_type, + is_nullable, + ordinal_position + from information_schema.columns + where table_schema not in ('pg_catalog', 'information_schema') + order by table_schema, table_name, ordinal_position + `); + + type ColumnInfo = { + name: string; + type: string; + nullable: boolean; + position: number; + }; + + type TableInfo = { + schema: string; + name: string; + columns: ColumnInfo[]; + }; + + const tableMap = new Map(); + const rows = (schemaRows as unknown as { rows?: unknown[] })?.rows ?? []; + for (const row of rows as Array>) { + const tableSchema = String(row.table_schema ?? ''); + const tableName = String(row.table_name ?? ''); + if (!tableSchema || !tableName) continue; + + const key = `${tableSchema}.${tableName}`; + const existing = tableMap.get(key); + const tableInfo: TableInfo = existing ?? { + schema: tableSchema, + name: tableName, + columns: [], + }; + + tableInfo.columns.push({ + name: String(row.column_name ?? ''), + type: String(row.data_type ?? ''), + nullable: String(row.is_nullable ?? '') === 'YES', + position: Number(row.ordinal_position ?? 0), + }); + + if (!existing) tableMap.set(key, tableInfo); + } + + const tables = [...tableMap.values()]; return { statusCode: 200, - body: JSON.stringify({ success: true, result }), + body: JSON.stringify({ success: true, ping, schema: { tables } }), }; } catch (error) { const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + const maybeAny = error as unknown as { cause?: unknown; code?: unknown }; + const causeMessage = + maybeAny?.cause instanceof Error + ? maybeAny.cause.message + : typeof maybeAny?.cause === 'string' + ? maybeAny.cause + : undefined; + const causeCode = + typeof (maybeAny?.cause as { code?: unknown } | undefined)?.code === 'string' + ? (maybeAny.cause as { code?: string }).code + : typeof maybeAny?.code === 'string' + ? maybeAny.code + : undefined; return { statusCode: 500, - body: JSON.stringify({ success: false, error: errorMessage }), + body: JSON.stringify({ + success: false, + error: errorMessage, + cause: causeMessage, + code: causeCode, + }), }; } }; diff --git a/serverless/src/lib/db.ts b/serverless/src/lib/db.ts index a4f8118..e7346bc 100644 --- a/serverless/src/lib/db.ts +++ b/serverless/src/lib/db.ts @@ -5,8 +5,8 @@ import { Pool } from 'pg'; import * as schema from '../db/schema'; type DbSecret = { - username: string; - password: string; + username: string; + password: string; }; let cachedSecret: DbSecret | undefined; @@ -14,62 +14,65 @@ let cachedPool: Pool | undefined; let cachedDb: NodePgDatabase | undefined; function requireEnv(name: string): string { - const value = process.env[name]; - if (!value) { - throw new Error(`Missing required env var: ${name}`); - } - return value; + const value = process.env[name]; + if (!value) { + throw new Error(`Missing required env var: ${name}`); + } + return value; } async function getDbSecret(): Promise { - if (cachedSecret) return cachedSecret; + if (cachedSecret) return cachedSecret; - const secretArn = requireEnv('DB_SECRET_ARN'); - const client = new SecretsManagerClient({}); - const result = await client.send( - new GetSecretValueCommand({ - SecretId: secretArn, - }), - ); + const secretArn = requireEnv('DB_SECRET_ARN'); + const client = new SecretsManagerClient({}); + const result = await client.send( + new GetSecretValueCommand({ + SecretId: secretArn, + }) + ); - if (!result.SecretString) { - throw new Error('SecretString was empty for DB_SECRET_ARN'); - } + if (!result.SecretString) { + throw new Error('SecretString was empty for DB_SECRET_ARN'); + } - const parsed = JSON.parse(result.SecretString) as Partial; - if (!parsed.username || !parsed.password) { - throw new Error('DB secret missing username/password'); - } + const parsed = JSON.parse(result.SecretString) as Partial; + if (!parsed.username || !parsed.password) { + throw new Error('DB secret missing username/password'); + } - cachedSecret = { username: parsed.username, password: parsed.password }; - return cachedSecret; + cachedSecret = { username: parsed.username, password: parsed.password }; + return cachedSecret; } export async function getPool(): Promise { - if (cachedPool) return cachedPool; + if (cachedPool) return cachedPool; - const host = requireEnv('DB_HOST'); - const port = Number.parseInt(requireEnv('DB_PORT'), 10); - const database = requireEnv('DB_NAME'); - const { username: user, password } = await getDbSecret(); + const host = requireEnv('DB_HOST'); + const port = Number.parseInt(requireEnv('DB_PORT'), 10); + const database = requireEnv('DB_NAME'); + const { username: user, password } = await getDbSecret(); - cachedPool = new Pool({ - host, - port, - database, - user, - password, - max: 5, - idleTimeoutMillis: 30_000, - connectionTimeoutMillis: 10_000, - }); + cachedPool = new Pool({ + host, + port, + database, + user, + password, + ssl: { + rejectUnauthorized: true, + }, + max: 5, + idleTimeoutMillis: 30_000, + connectionTimeoutMillis: 10_000, + }); - return cachedPool; + return cachedPool; } export async function getDb(): Promise> { - if (cachedDb) return cachedDb; - const pool = await getPool(); - cachedDb = drizzle(pool, { schema }); - return cachedDb; + if (cachedDb) return cachedDb; + const pool = await getPool(); + cachedDb = drizzle(pool, { schema }); + return cachedDb; }