From 3dfd9ea3c6da15d690acdfbf83d50460e54e9553 Mon Sep 17 00:00:00 2001 From: BHI Staging Agent Date: Sun, 5 Apr 2026 20:15:36 +0000 Subject: [PATCH] BHI layer v1: docs, schema, Phase A ingestion stubs --- docs/integration_plan.md | 156 ++++++++++ docs/scoring.md | 125 ++++++++ docs/sources.md | 273 ++++++++++++++++++ docs/target_questions.md | 64 ++++ .../__pycache__/_common.cpython-312.pyc | Bin 0 -> 7850 bytes jobs/ingestion/_common.py | 146 ++++++++++ jobs/ingestion/bls_oes.py | 93 ++++++ jobs/ingestion/cdc_brfss.py | 92 ++++++ jobs/ingestion/cdc_wonder_mortality.py | 119 ++++++++ jobs/ingestion/cdc_yrbss.py | 95 ++++++ jobs/ingestion/cms_hospital_compare.py | 77 +++++ jobs/ingestion/cms_ipfqr.py | 137 +++++++++ jobs/ingestion/cms_nursing_home.py | 82 ++++++ jobs/ingestion/cms_pos.py | 143 +++++++++ jobs/ingestion/hrsa_hpsa.py | 85 ++++++ jobs/ingestion/idea_part_b.py | 93 ++++++ jobs/ingestion/nppes.py | 114 ++++++++ jobs/ingestion/nsch.py | 96 ++++++ jobs/ingestion/samhsa_locator.py | 95 ++++++ jobs/ingestion/samhsa_nssats_nmhss.py | 102 +++++++ schemas/bhi_tables.sql | 212 ++++++++++++++ 21 files changed, 2399 insertions(+) create mode 100644 docs/integration_plan.md create mode 100644 docs/scoring.md create mode 100644 docs/sources.md create mode 100644 docs/target_questions.md create mode 100644 jobs/ingestion/__pycache__/_common.cpython-312.pyc create mode 100644 jobs/ingestion/_common.py create mode 100644 jobs/ingestion/bls_oes.py create mode 100644 jobs/ingestion/cdc_brfss.py create mode 100644 jobs/ingestion/cdc_wonder_mortality.py create mode 100644 jobs/ingestion/cdc_yrbss.py create mode 100644 jobs/ingestion/cms_hospital_compare.py create mode 100644 jobs/ingestion/cms_ipfqr.py create mode 100644 jobs/ingestion/cms_nursing_home.py create mode 100644 jobs/ingestion/cms_pos.py create mode 100644 jobs/ingestion/hrsa_hpsa.py create mode 100644 jobs/ingestion/idea_part_b.py create mode 100644 jobs/ingestion/nppes.py create mode 100644 jobs/ingestion/nsch.py create mode 100644 jobs/ingestion/samhsa_locator.py create mode 100644 jobs/ingestion/samhsa_nssats_nmhss.py create mode 100644 schemas/bhi_tables.sql diff --git a/docs/integration_plan.md b/docs/integration_plan.md new file mode 100644 index 0000000..d3f05bd --- /dev/null +++ b/docs/integration_plan.md @@ -0,0 +1,156 @@ +# BHI Layer — Integration Plan + +Steps to merge the BHI layer into the base Economic Brain after the base build finishes. + +**Prereqs** (verified before step 1): +- Base Brain is running: `psql -d brain -c '\dt'` shows core tables including `job_runs`. +- `/home/ubuntu/economic-brain/` contains a working `jobs/` directory structure. +- DATABASE_URL env var exported and pointing at the `brain` Postgres. + +--- + +## 1. Apply the BHI schema + +```bash +cd /home/ubuntu/economic-brain-bhi +psql "$DATABASE_URL" -f schemas/bhi_tables.sql +psql "$DATABASE_URL" -c "\dt bhi_*" +# Expect 9 tables: bhi_facilities, bhi_facility_quality, bhi_facility_financials, +# bhi_demand_indicators, bhi_workforce, bhi_shortages, bhi_rtf_licensing, +# bhi_policy_events, bhi_crisis_calls +``` + +## 2. Copy ingestion jobs into the Brain's jobs tree + +```bash +mkdir -p /home/ubuntu/economic-brain/jobs/bhi +cp /home/ubuntu/economic-brain-bhi/jobs/ingestion/*.py /home/ubuntu/economic-brain/jobs/bhi/ +# _common.py is included; it reads DATABASE_URL from env already +``` + +Install Python deps if the base Brain doesn't already have them: + +```bash +pip install requests psycopg2-binary +``` + +## 3. Smoke test every Phase A job (no DB writes) + +```bash +cd /home/ubuntu/economic-brain/jobs/bhi +for f in cms_ipfqr.py cms_hospital_compare.py cms_nursing_home.py \ + samhsa_locator.py hrsa_hpsa.py nppes.py cdc_brfss.py \ + cdc_yrbss.py cdc_wonder_mortality.py bls_oes.py cms_pos.py \ + samhsa_nssats_nmhss.py idea_part_b.py nsch.py; do + echo "=== $f ===" + python3 "$f" test || echo "FAIL: $f" +done +``` + +Every job should print `OK:` and exit 0. If any fail, fix the endpoint/URL in the job file before proceeding. + +## 4. Run jobs in dependency order + +```bash +# Facilities first (feed bhi_facilities.facility_id FK for quality/financials) +python3 cms_ipfqr.py +python3 cms_hospital_compare.py +python3 cms_nursing_home.py +python3 samhsa_locator.py +python3 cms_pos.py +python3 samhsa_nssats_nmhss.py +python3 nppes.py + +# Shortages + demand (independent) +python3 hrsa_hpsa.py +python3 cdc_wonder_mortality.py +python3 cdc_brfss.py +python3 cdc_yrbss.py +python3 idea_part_b.py +python3 nsch.py + +# Workforce +python3 bls_oes.py +``` + +Monitor `job_runs`: +```sql +SELECT job_name, status, started_at, finished_at, error +FROM job_runs WHERE job_name LIKE 'bhi_%' ORDER BY started_at DESC; +``` + +## 5. Import n8n workflows (scheduled refresh) + +Create workflows in n8n (or add to existing scheduler): + +| Workflow | Cron | Script | +|---|---|---| +| BHI: CMS facilities refresh | `0 3 * * 1` (weekly Mon 3am) | `cms_ipfqr.py`, `cms_hospital_compare.py`, `cms_nursing_home.py` | +| BHI: SAMHSA locator refresh | `0 4 1 * *` (monthly) | `samhsa_locator.py` | +| BHI: HRSA HPSA refresh | `0 5 * * 2` (weekly Tue 5am) | `hrsa_hpsa.py` | +| BHI: CDC demand refresh | `0 6 1 * *` (monthly) | `cdc_brfss.py`, `cdc_yrbss.py`, `cdc_wonder_mortality.py` | +| BHI: Workforce refresh | `0 7 1 */3 *` (quarterly) | `bls_oes.py` | +| BHI: CMS POS refresh | `0 8 1 */3 *` (quarterly) | `cms_pos.py` | + +Workflow template: Cron node -> Execute Command (`python3 /home/ubuntu/economic-brain/jobs/bhi/