diff --git a/db/01-load-schema.sh b/db/01-load-schema.sh new file mode 100755 index 0000000..2ab5245 --- /dev/null +++ b/db/01-load-schema.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Usage: SSH_PASS= bash db/01-load-schema.sh +# Or: configure SSH key auth first, then just: bash db/01-load-schema.sh +set -e + +SERVER="setia@ims.setia.com.my" +PORT=9321 + +echo "==> Copying schema files to server..." +scp -P "$PORT" db/schema.sql db/rls.sql "$SERVER:/tmp/" + +echo "==> Loading schema on server..." +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -f /tmp/schema.sql && sudo -u postgres psql -d ims -f /tmp/rls.sql" + +echo "==> Cleaning up temp files..." +ssh -p "$PORT" "$SERVER" "rm /tmp/schema.sql /tmp/rls.sql" + +echo "==> Schema loaded. Verifying table count..." +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS table_count FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE';\"" +# Expected: 14 tables (13 original + password_reset_tokens) diff --git a/db/02-dump-and-restore.sh b/db/02-dump-and-restore.sh new file mode 100755 index 0000000..83da334 --- /dev/null +++ b/db/02-dump-and-restore.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/02-dump-and-restore.sh +set -e + +if [ -z "$SUPABASE_DB_URL" ]; then + echo "ERROR: SUPABASE_DB_URL not set" + exit 1 +fi + +SERVER="setia@ims.setia.com.my" +PORT=9321 +DUMP_FILE="/tmp/ims-supabase-dump-$(date +%Y%m%d-%H%M%S).sql" + +echo "==> Dumping Supabase public schema (data only)..." +pg_dump \ + --data-only \ + --no-owner \ + --no-privileges \ + --schema=public \ + --exclude-table=schema_migrations \ + "$SUPABASE_DB_URL" \ + -f "$DUMP_FILE" + +echo "==> Dump written to $DUMP_FILE ($(du -sh "$DUMP_FILE" | cut -f1))" + +echo "==> Copying dump to server..." +scp -P "$PORT" "$DUMP_FILE" "$SERVER:/tmp/ims-data.sql" + +echo "==> Restoring on server..." +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -f /tmp/ims-data.sql 2>&1 | tail -20" + +echo "==> Cleaning up..." +ssh -p "$PORT" "$SERVER" "rm /tmp/ims-data.sql" +rm "$DUMP_FILE" + +echo "==> Done. Run db/05-verify.sh to check row counts." diff --git a/db/03-migrate-passwords.sh b/db/03-migrate-passwords.sh new file mode 100755 index 0000000..5bd1ebd --- /dev/null +++ b/db/03-migrate-passwords.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/03-migrate-passwords.sh +set -e + +if [ -z "$SUPABASE_DB_URL" ]; then + echo "ERROR: SUPABASE_DB_URL not set" + exit 1 +fi + +SERVER="setia@ims.setia.com.my" +PORT=9321 +AUTH_DUMP="/tmp/ims-auth-users-$(date +%Y%m%d-%H%M%S).csv" + +echo "==> Exporting auth.users (id + encrypted_password + created_at) from Supabase..." +psql "$SUPABASE_DB_URL" -c "\COPY (SELECT id, encrypted_password, created_at FROM auth.users WHERE encrypted_password IS NOT NULL) TO STDOUT WITH CSV HEADER" > "$AUTH_DUMP" + +echo "==> Exported $(wc -l < "$AUTH_DUMP") rows (including header)" + +echo "==> Copying to server..." +scp -P "$PORT" "$AUTH_DUMP" "$SERVER:/tmp/auth-users.csv" + +echo "==> Applying password hashes to users table on server..." +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims" <<'ENDSQL' +-- Create temp table for the auth data +CREATE TEMP TABLE auth_users_import ( + id UUID, + password_hash TEXT, + created_at TIMESTAMPTZ +); + +\COPY auth_users_import FROM '/tmp/auth-users.csv' WITH CSV HEADER; + +-- Update users table +UPDATE users u +SET + password_hash = a.password_hash, + email_verified_at = COALESCE(u.email_verified_at, a.created_at) +FROM auth_users_import a +WHERE u.id = a.id; + +SELECT count(*) AS updated_users FROM users WHERE password_hash IS NOT NULL; +ENDSQL + +echo "==> Verifying no NULL password_hash remains..." +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS null_password_users FROM users WHERE password_hash IS NULL;\"" +# Should be 0 + +echo "==> Cleaning up..." +ssh -p "$PORT" "$SERVER" "rm /tmp/auth-users.csv" +rm "$AUTH_DUMP" + +echo "==> Password migration complete." diff --git a/db/04-migrate-evidence.sh b/db/04-migrate-evidence.sh new file mode 100755 index 0000000..ee4f471 --- /dev/null +++ b/db/04-migrate-evidence.sh @@ -0,0 +1,77 @@ +#!/bin/bash +# Usage: +# SUPABASE_URL="https://xxx.supabase.co" \ +# SUPABASE_SERVICE_ROLE_KEY="eyJ..." \ +# bash db/04-migrate-evidence.sh +set -e + +if [ -z "$SUPABASE_URL" ] || [ -z "$SUPABASE_SERVICE_ROLE_KEY" ]; then + echo "ERROR: SUPABASE_URL and SUPABASE_SERVICE_ROLE_KEY must be set" + exit 1 +fi + +SERVER="setia@ims.setia.com.my" +PORT=9321 +REMOTE_DIR="/var/lib/ims/evidence" +LOCAL_TMP="/tmp/ims-evidence-migration" + +mkdir -p "$LOCAL_TMP" + +echo "==> Listing all objects in Supabase Storage bucket 'evidence'..." +# Paginate: offset 0, limit 1000 (repeat if >1000 files) +OBJECTS=$(curl -s \ + -H "Authorization: Bearer $SUPABASE_SERVICE_ROLE_KEY" \ + -H "Content-Type: application/json" \ + "$SUPABASE_URL/storage/v1/object/list/evidence" \ + -d '{"limit": 1000, "offset": 0, "prefix": ""}') + +COUNT=$(echo "$OBJECTS" | python3 -c "import sys,json; data=json.load(sys.stdin); print(len(data))" 2>/dev/null || echo 0) +echo "==> Found $COUNT objects" + +if [ "$COUNT" -eq 0 ]; then + echo "==> No evidence files to migrate. Exiting." + exit 0 +fi + +echo "==> Downloading evidence files..." +# Export vars so the Python subprocess can access them +export LOCAL_TMP SERVER PORT REMOTE_DIR +echo "$OBJECTS" | python3 - <<'PYEOF' +import sys, json, os, subprocess, urllib.request + +objects = json.load(sys.stdin) +supabase_url = os.environ['SUPABASE_URL'] +service_role_key = os.environ['SUPABASE_SERVICE_ROLE_KEY'] +local_tmp = os.environ.get('LOCAL_TMP', '/tmp/ims-evidence-migration') +server = os.environ['SERVER'] +port = os.environ['PORT'] +remote_dir = os.environ['REMOTE_DIR'] + +for i, obj in enumerate(objects): + name = obj['name'] # e.g. "userId/incidentId/stage/filename.jpg" + local_path = os.path.join(local_tmp, name) + os.makedirs(os.path.dirname(local_path), exist_ok=True) + + # Download from Supabase Storage + url = f"{supabase_url}/storage/v1/object/evidence/{name}" + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {service_role_key}"}) + with urllib.request.urlopen(req) as resp, open(local_path, 'wb') as f: + f.write(resp.read()) + + print(f"[{i+1}/{len(objects)}] Downloaded: {name}") +PYEOF + +echo "==> Uploading evidence files to server..." +# Create remote directory structure +ssh -p "$PORT" "$SERVER" "mkdir -p $REMOTE_DIR" + +# rsync preserves directory structure +rsync -az --progress -e "ssh -p $PORT" "$LOCAL_TMP/" "$SERVER:$REMOTE_DIR/" + +echo "==> Cleaning up local temp..." +rm -rf "$LOCAL_TMP" + +echo "==> Verifying file count on server..." +ssh -p "$PORT" "$SERVER" "find $REMOTE_DIR -type f | wc -l" + +echo "==> Evidence migration complete." diff --git a/db/05-verify.sh b/db/05-verify.sh new file mode 100755 index 0000000..55c91a3 --- /dev/null +++ b/db/05-verify.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Usage: SUPABASE_DB_URL="postgresql://postgres:..." bash db/05-verify.sh +set -e + +SERVER="setia@ims.setia.com.my" +PORT=9321 + +echo "=== IMS Data Migration Verification ===" +echo "" + +echo "--- Table row counts on NEW server ---" +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims" <<'ENDSQL' +SELECT + tablename, + n_live_tup AS row_count +FROM pg_stat_user_tables +ORDER BY tablename; +ENDSQL + +if [ -n "$SUPABASE_DB_URL" ]; then + echo "" + echo "--- Table row counts on SUPABASE (source) ---" + psql "$SUPABASE_DB_URL" <<'ENDSQL' +SELECT + tablename, + n_live_tup AS row_count +FROM pg_stat_user_tables +WHERE schemaname = 'public' +ORDER BY tablename; +ENDSQL +fi + +echo "" +echo "--- Users with NULL password_hash (should be 0) ---" +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) FROM users WHERE password_hash IS NULL;\"" + +echo "" +echo "--- Evidence files row count vs filesystem count ---" +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS db_evidence_rows FROM evidence_files WHERE deleted = false;\"" +ssh -p "$PORT" "$SERVER" "find /var/lib/ims/evidence -type f | wc -l" + +echo "" +echo "--- pgvector embeddings check ---" +ssh -p "$PORT" "$SERVER" "sudo -u postgres psql -d ims -c \"SELECT count(*) AS incidents_with_embedding FROM incidents WHERE embedding IS NOT NULL;\"" + +echo "" +echo "=== Verification complete. Compare counts above. ===" diff --git a/db/schema.sql b/db/schema.sql index 7b0f253..0ad7c08 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -478,6 +478,21 @@ VALUES ('00000000-0000-0000-0000-000000000012'::UUID, '00000000-0000-0000-0000-000000000001'::UUID, 'Loading Bay', 'scw1-loading-bay-qr-2026') ON CONFLICT (id) DO NOTHING; +-- --------------------------------------------------------------------------- +-- Password reset tokens (Phase 3 — custom auth) +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS password_reset_tokens ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + expires_at TIMESTAMPTZ NOT NULL, + used_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS password_reset_tokens_user_id_idx ON password_reset_tokens(user_id); + -- Seed: app_settings (final state after all migrations) INSERT INTO app_settings (key, value) VALUES ('DEEPSEEK_API_KEY', ''),