diff --git a/.github/workflows/deploy-custom-prod.yml b/.github/workflows/deploy-custom-prod.yml index 2803d2537..467cf709b 100644 --- a/.github/workflows/deploy-custom-prod.yml +++ b/.github/workflows/deploy-custom-prod.yml @@ -45,6 +45,7 @@ jobs: cost_profile: DEFAULT job_files: >- job_spec/AK_FIRE_SAFE.yml + job_spec/FIRE_TRACK.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r8id.xlarge,r8id.2xlarge,r8id.4xlarge,r8id.8xlarge,r8id.12xlarge,r8id.16xlarge default_max_vcpus: 640 @@ -286,28 +287,6 @@ jobs: same_account_publishing: false ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id - - environment: hyp3-carter - domain: hyp3-carter.asf.alaska.edu - template_bucket: cf-templates-1qx2mwia5g4kh-us-west-2 - image_tag: latest - product_lifetime_in_days: 30 - default_credits_per_user: 0 - default_application_status: APPROVED - cost_profile: DEFAULT - job_files: >- - job_spec/AUTORIFT.yml - job_spec/INSAR_GAMMA.yml - job_spec/RTC_GAMMA.yml - job_spec/INSAR_ISCE_BURST.yml - job_spec/INSAR_ISCE_MULTI_BURST.yml - instance_types: r8id.xlarge,r8id.2xlarge,r8id.4xlarge,r8id.8xlarge,r8id.12xlarge,r8id.16xlarge - default_max_vcpus: 640 - expanded_max_vcpus: 640 - required_surplus: 0 - security_environment: ASF - same_account_publishing: false - ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id - - environment: hyp3-lavas domain: hyp3-lavas.asf.alaska.edu template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 @@ -333,7 +312,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-custom-test.yml b/.github/workflows/deploy-custom-test.yml index 59697a84e..9be165093 100644 --- a/.github/workflows/deploy-custom-test.yml +++ b/.github/workflows/deploy-custom-test.yml @@ -89,6 +89,7 @@ jobs: cost_profile: DEFAULT job_files: >- job_spec/AK_FIRE_SAFE.yml + job_spec/FIRE_TRACK.yml job_spec/INSAR_ISCE_BURST.yml instance_types: r8id.xlarge,r8id.2xlarge,r8id.4xlarge,r8id.8xlarge,r8id.12xlarge,r8id.16xlarge default_max_vcpus: 640 @@ -183,7 +184,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-prod.yml b/.github/workflows/deploy-daac-prod.yml index 8f7c1f8f8..74851d4cb 100644 --- a/.github/workflows/deploy-daac-prod.yml +++ b/.github/workflows/deploy-daac-prod.yml @@ -47,7 +47,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-daac-test.yml b/.github/workflows/deploy-daac-test.yml index da870b13e..c40ac3495 100644 --- a/.github/workflows/deploy-daac-test.yml +++ b/.github/workflows/deploy-daac-test.yml @@ -47,7 +47,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-edc-sandbox.yml b/.github/workflows/deploy-edc-sandbox.yml index 37ae01bf0..2c159d30d 100644 --- a/.github/workflows/deploy-edc-sandbox.yml +++ b/.github/workflows/deploy-edc-sandbox.yml @@ -45,7 +45,7 @@ jobs: name: ${{ matrix.environment }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-plus-prod.yml b/.github/workflows/deploy-plus-prod.yml index a5b30b940..4ed4ecc3a 100644 --- a/.github/workflows/deploy-plus-prod.yml +++ b/.github/workflows/deploy-plus-prod.yml @@ -45,7 +45,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/deploy-plus-test.yml b/.github/workflows/deploy-plus-test.yml index 93f669b10..71951371c 100644 --- a/.github/workflows/deploy-plus-test.yml +++ b/.github/workflows/deploy-plus-test.yml @@ -46,7 +46,7 @@ jobs: url: https://${{ matrix.domain }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 0 diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml index e966d66c5..c7b99efdb 100644 --- a/.github/workflows/static-analysis.yml +++ b/.github/workflows/static-analysis.yml @@ -29,7 +29,7 @@ jobs: matrix: security_environment: [ASF, EDC, JPL, JPL-public] steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -43,7 +43,7 @@ jobs: openapi-spec-validator: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: 3.13 @@ -56,7 +56,7 @@ jobs: statelint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: ruby/setup-ruby@v1 with: ruby-version: 2.7 @@ -77,7 +77,7 @@ jobs: runs-on: ubuntu-latest if: ${{ ! github.event.pull_request.head.repo.fork }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: snyk/actions/setup@v1.0.0 - uses: actions/setup-python@v6 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 599861f29..c787b8561 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 32206f397..ef6f6a1e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [10.17.6] + +### Added +- Added RDS Connection Permission to the TaskRole. +- Added `FIRE_TRACK` job spec to `hyp3-ak-fire-safe`. +- Added parameter `upload_to_db` to `AK_FIRE_SAFE` job spec. +- Added environment variable `DB_HOST` to `AK_FIRE_SAFE` job spec. + +### Changed +- Parquet file products are now visible. +- Sentinel-1D scenes are now accepted by `ARIA_S1_COSEIS`, `INSAR_ISCE`, `WATER_MAP` and `WATERMAP_EQ` jobs. +- Updated the pinned version of hyp3-isce2 from v3.0.1 to v3.0.2 in `INSAR_ISCE_BURST`, `INSAR_ISCE_BURST`, and `INSAR_ISCE_BURST` job specs to support Sentinel-1D processing in single burst jobs. + +### Fixed +- The SrgGslc and SlimSAR compute environments have been upgraded to AL2023-based AMIs from AL2 AMIs due to the pending end-of-life of AL2. +- The `check_same_relative_orbits` validator has been updated to account for the repositioning of Sentinel-1C in June, 2026. + +### Removed +- The `hyp3-carter` deployment has been retired. + + ## [10.17.5] ### Added diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py index 161d8ea1f..6218cd10c 100644 --- a/apps/api/src/hyp3_api/validation.py +++ b/apps/api/src/hyp3_api/validation.py @@ -207,7 +207,7 @@ def check_same_relative_orbits(_, granule_metadata: list[dict]) -> None: previous_relative_orbit = None for granule in granule_metadata: name_split = granule['name'].split('_') - absolute_orbit = name_split[7] + absolute_orbit = int(name_split[7]) # "Relationship between relative and absolute orbit numbers": https://sentiwiki.copernicus.eu/web/s1-products mission = name_split[0] if mission == 'S1A': @@ -215,12 +215,15 @@ def check_same_relative_orbits(_, granule_metadata: list[dict]) -> None: elif mission == 'S1B': offset = 27 elif mission == 'S1C': - offset = 172 + if absolute_orbit <= 8018: + offset = 172 + else: + offset = 99 elif mission == 'S1D': offset = 42 else: raise ValueError(f'Encountered unknown Sentinel-1 mission: {mission}') - relative_orbit = ((int(absolute_orbit) - offset) % 175) + 1 + relative_orbit = ((absolute_orbit - offset) % 175) + 1 if not previous_relative_orbit: previous_relative_orbit = relative_orbit if relative_orbit != previous_relative_orbit: diff --git a/apps/compute-cf.yml.j2 b/apps/compute-cf.yml.j2 index ed40f3bc7..33f73deac 100644 --- a/apps/compute-cf.yml.j2 +++ b/apps/compute-cf.yml.j2 @@ -193,6 +193,9 @@ Resources: - Effect: Allow Action: sns:Publish Resource: arn:aws:sns:* + - Effect: Allow + Action: rds-db:connect + Resource: !Sub "arn:aws:rds-db:${AWS::Region}:${AWS::AccountId}:dbuser:*/*" BatchServiceRole: Type: {{ 'Custom::JplRole' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::Role' }} diff --git a/apps/get-files/src/get_files.py b/apps/get-files/src/get_files.py index d2edfd292..d42d76b2a 100644 --- a/apps/get-files/src/get_files.py +++ b/apps/get-files/src/get_files.py @@ -42,7 +42,7 @@ def get_object_file_type(bucket: str, key: str) -> str | None: def visible_product(product_path: str | Path) -> bool: - return Path(product_path).suffix in ('.zip', '.nc', '.geojson') + return Path(product_path).suffix in ('.zip', '.nc', '.geojson', '.parquet') def get_products(files: list[dict]) -> list[dict]: diff --git a/job_spec/AK_FIRE_SAFE.yml b/job_spec/AK_FIRE_SAFE.yml index da825d312..33a56b2bd 100644 --- a/job_spec/AK_FIRE_SAFE.yml +++ b/job_spec/AK_FIRE_SAFE.yml @@ -36,6 +36,11 @@ AK_FIRE_SAFE: default: null format: date example: "2025-06-01" + upload_to_db: + api_schema: + default: false + type: boolean + description: Add the data to the AWS Aurora database. input_bucket: api_schema: description: Bucket that contains fire detection products. @@ -69,6 +74,8 @@ AK_FIRE_SAFE: - Ref::end_date - --extent - Ref::extent + - --upload-to-db + - Ref::upload_to_db - --input-bucket - Ref::input_bucket - --input-prefix @@ -80,3 +87,4 @@ AK_FIRE_SAFE: secrets: - EARTHDATA_USERNAME - EARTHDATA_PASSWORD + - DB_HOST diff --git a/job_spec/ARIA_S1_COSEIS.yml b/job_spec/ARIA_S1_COSEIS.yml index 48e27d0ee..52fb7d814 100644 --- a/job_spec/ARIA_S1_COSEIS.yml +++ b/job_spec/ARIA_S1_COSEIS.yml @@ -11,7 +11,7 @@ ARIA_S1_COSEIS: items: description: The name of the Sentinel-1 SLC granules to use as reference scenes for InSAR processing type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD]V" + pattern: "^S1[ABCD]_IW_SLC__1S[SD]V" minLength: 67 maxLength: 67 example: S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 @@ -22,7 +22,7 @@ ARIA_S1_COSEIS: items: description: The name of the Sentinel-1 SLC granules to use as secondary scenes for InSAR processing type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD]V" + pattern: "^S1[ABCD]_IW_SLC__1S[SD]V" minLength: 67 maxLength: 67 example: S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 diff --git a/job_spec/FIRE_TRACK.yml b/job_spec/FIRE_TRACK.yml new file mode 100644 index 000000000..b9271894e --- /dev/null +++ b/job_spec/FIRE_TRACK.yml @@ -0,0 +1,41 @@ +FIRE_TRACK: + required_parameters: + - input_bucket + - input_prefix + parameters: + input_bucket: + api_schema: + description: Bucket that contains fire detection products. + type: string + example: "ak-fire-safe-data-test" + enum: + - "ak-fire-safe-data" + - "ak-fire-safe-data-test" + input_prefix: + api_schema: + description: Prefix to pull the fire detection products. + type: string + example: "txt" + cost_profiles: + DEFAULT: + cost: 1.0 + validators: [] + steps: + - name: '' + image: 761639967505.dkr.ecr.us-west-2.amazonaws.com/hyp3-fire-tracking + command: + - --bucket + - Ref::bucket + - --bucket-prefix + - Ref::bucket_prefix + - --input-bucket + - Ref::input_bucket + - --input-prefix + - Ref::input_prefix + timeout: 5400 + compute_environment: Default + vcpu: 1 + memory: 7600 + secrets: + - EARTHDATA_USERNAME + - EARTHDATA_PASSWORD diff --git a/job_spec/INSAR_ISCE.yml b/job_spec/INSAR_ISCE.yml index 93daeb372..eae72b87d 100644 --- a/job_spec/INSAR_ISCE.yml +++ b/job_spec/INSAR_ISCE.yml @@ -11,7 +11,7 @@ INSAR_ISCE: items: description: The name of the Sentinel-1 SLC granules to use as reference scenes for InSAR processing type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD]V" + pattern: "^S1[ABCD]_IW_SLC__1S[SD]V" minLength: 67 maxLength: 67 example: S1B_IW_SLC__1SDV_20210723T014947_20210723T015014_027915_0354B4_B3A9 @@ -22,7 +22,7 @@ INSAR_ISCE: items: description: The name of the Sentinel-1 SLC granules to use as secondary scenes for InSAR processing type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD]V" + pattern: "^S1[ABCD]_IW_SLC__1S[SD]V" minLength: 67 maxLength: 67 example: S1B_IW_SLC__1SDV_20210711T014947_20210711T015013_027740_034F80_D404 diff --git a/job_spec/INSAR_ISCE_BURST.yml b/job_spec/INSAR_ISCE_BURST.yml index 831f83799..31f383268 100644 --- a/job_spec/INSAR_ISCE_BURST.yml +++ b/job_spec/INSAR_ISCE_BURST.yml @@ -43,7 +43,7 @@ INSAR_ISCE_BURST: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-isce2 - image_tag: 3.0.1 + image_tag: 3.0.2 command: - ++process - insar_tops_burst diff --git a/job_spec/INSAR_ISCE_BURST_EC2.yml b/job_spec/INSAR_ISCE_BURST_EC2.yml index ac947735e..6fb7189c0 100644 --- a/job_spec/INSAR_ISCE_BURST_EC2.yml +++ b/job_spec/INSAR_ISCE_BURST_EC2.yml @@ -43,7 +43,7 @@ INSAR_ISCE_BURST_EC2: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-isce2 - image_tag: 3.0.1 + image_tag: 3.0.2 command: - ++process - insar_tops_burst diff --git a/job_spec/INSAR_ISCE_BURST_SPOT.yml b/job_spec/INSAR_ISCE_BURST_SPOT.yml index 9dca56f76..9b050e78b 100644 --- a/job_spec/INSAR_ISCE_BURST_SPOT.yml +++ b/job_spec/INSAR_ISCE_BURST_SPOT.yml @@ -43,7 +43,7 @@ INSAR_ISCE_BURST: steps: - name: '' image: ghcr.io/asfhyp3/hyp3-isce2 - image_tag: 3.0.1 + image_tag: 3.0.2 command: - ++process - insar_tops_burst diff --git a/job_spec/WATER_MAP.yml b/job_spec/WATER_MAP.yml index 64c5f3d5e..8821eeb54 100644 --- a/job_spec/WATER_MAP.yml +++ b/job_spec/WATER_MAP.yml @@ -11,13 +11,13 @@ WATER_MAP: anyOf: - description: The name of the IW VV+VH Sentinel-1 GRDH granule to process type: string - pattern: "^S1[ABC]_IW_GRDH_1SDV" + pattern: "^S1[ABCD]_IW_GRDH_1SDV" minLength: 67 maxLength: 67 example: S1A_IW_GRDH_1SDV_20210413T235641_20210413T235706_037439_0469D0_3F2B - description: The name of the IW VV+VH Sentinel-1 SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1SDV" + pattern: "^S1[ABCD]_IW_SLC__1SDV" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SDV_20211110T234815_20211110T234842_040516_04CE0A_E717 diff --git a/job_spec/WATER_MAP_EQ.yml b/job_spec/WATER_MAP_EQ.yml index f3e71f194..c849b86f3 100644 --- a/job_spec/WATER_MAP_EQ.yml +++ b/job_spec/WATER_MAP_EQ.yml @@ -11,13 +11,13 @@ WATER_MAP_EQ: anyOf: - description: The name of the IW VV+VH Sentinel-1 GRDH granule to process type: string - pattern: "^S1[ABC]_IW_GRDH_1S[SD]V" + pattern: "^S1[ABCD]_IW_GRDH_1S[SD]V" minLength: 67 maxLength: 67 example: S1A_IW_GRDH_1SDV_20210413T235641_20210413T235706_037439_0469D0_3F2B - description: The name of the IW VV+VH Sentinel-1 SLC granule to process type: string - pattern: "^S1[ABC]_IW_SLC__1S[SD]V" + pattern: "^S1[ABCD]_IW_SLC__1S[SD]V" minLength: 67 maxLength: 67 example: S1A_IW_SLC__1SDV_20211110T234815_20211110T234842_040516_04CE0A_E717 diff --git a/job_spec/config/compute_environments.yml b/job_spec/config/compute_environments.yml index 5a9b98ffb..72fa012f4 100644 --- a/job_spec/config/compute_environments.yml +++ b/job_spec/config/compute_environments.yml @@ -7,7 +7,7 @@ compute_environments: # max_vcpus SrgGslc: instance_types: g6.2xlarge - ami_id: "'{{resolve:ssm:/aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id}}'" + ami_id: "'{{resolve:ssm:/aws/service/ecs/optimized-ami/amazon-linux-2023/gpu/recommended/image_id}}'" SrgTimeSeries: instance_types: c6id.16xlarge ItsLiveSpotIntel: @@ -21,7 +21,7 @@ compute_environments: ami_id: "'{{resolve:ssm:/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id}}'" SlimSAR: instance_types: g4dn.2xlarge,g4dn.4xlarge,g4dn.8xlarge,g4dn.16xlarge - ami_id: "'{{resolve:ssm:/aws/service/ecs/optimized-ami/amazon-linux-2/gpu/recommended/image_id}}'" + ami_id: "'{{resolve:ssm:/aws/service/ecs/optimized-ami/amazon-linux-2023/gpu/recommended/image_id}}'" DefaultSpot: allocation_type: SPOT allocation_strategy: SPOT_PRICE_CAPACITY_OPTIMIZED diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py index 0cc039d8c..663e1522f 100644 --- a/tests/test_api/test_validation.py +++ b/tests/test_api/test_validation.py @@ -459,6 +459,9 @@ def test_check_same_relative_orbits(): {'name': 'S1A_IW_RAW__0SDV_20200816T161620_20200816T161652_033934_03EFCE_5730'}, {'name': 'S1B_IW_RAW__0SDV_20200810T161537_20200810T161610_022863_02B66A_F7D7'}, {'name': 'S1B_IW_RAW__0SDV_20200623T161535_20200623T161607_022163_02A10F_7FD6'}, + {'name': 'S1C_IW_RAW__0SDV_20260604T170742_20260604T170815_007958_0102EF_499C'}, + {'name': 'S1C_IW_RAW__0SDV_20260625T232524_20260625T232557_008235_DDDDDD_CCCC'}, + {'name': 'S1D_IW_RAW__0SDV_20260617T171023_20260617T171056_003278_005BC8_55E8'}, ] invalid_granule_metadata = valid_granule_metadata.copy() invalid_granule_metadata.append({'name': 'S1B_IW_RAW__0SDV_20200623T161535_20200623T161607_012345_02A10F_7FD6'})