Repository navigation
64 lines (58 loc) · 1.99 KB
/
Copy pathdevelopments_datasync.yml
File metadata and controls
64 lines (58 loc) · 1.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
name: DevDB - Data Sync of HPD and DOB Data
on:
schedule:
- cron: "23 20 * * 0" # Sundays, 8:23pm
timezone: "America/New_York"
workflow_dispatch:
jobs:
sync:
name: Sync DOB BIS and HPD HNY
runs-on: ubuntu-22.04
container:
image: nycplanning/build-geosupport:latest
env:
RECIPES_BUCKET: edm-recipes
strategy:
matrix:
dataset:
- dob_bis_permits
- dob_bis_applications
- hpd_hny_units_by_building
steps:
- uses: actions/checkout@v7
- name: Install dcpy workspace
run: uv sync --all-packages
- name: Load Secrets
uses: 1password/load-secrets-action@v5
with:
export-env: true
env:
OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
AWS_S3_ENDPOINT: "op://Data Engineering/DO_keys/AWS_S3_ENDPOINT"
AWS_SECRET_ACCESS_KEY: "op://Data Engineering/DO_keys/AWS_SECRET_ACCESS_KEY"
AWS_ACCESS_KEY_ID: "op://Data Engineering/DO_keys/AWS_ACCESS_KEY_ID"
BUILD_ENGINE_SERVER: "op://Data Engineering/EDM_DATA/server_url"
- name: Finish container setup ...
run: ./bash/docker_container_setup.sh
- name: Ingest dataset
run: uv run python3 -m dcpy lifecycle ingest ingest ${{ matrix.dataset }} --latest --push
create_issue_on_failure:
permissions:
contents: read
issues: write
needs: sync
runs-on: ubuntu-22.04
if: ${{ failure() && (github.event_name == 'schedule') }}
steps:
- uses: actions/checkout@v7
with:
sparse-checkout: .github
- name: Create issue on failure
uses: JasonEtco/create-an-issue@v2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTION: ${{ github.workflow }}
BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
filename: .github/ISSUE_TEMPLATE/scheduled_action_failure.md
update_existing: true