Invoke sysops deployment with image tag for Live updates.

.github/workflows/ci-deploy-env.yml

@@ -43,7 +43,7 @@ jobs:
       uses: docker/setup-buildx-action@v2
 
     - name: Cache Docker layers
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: cache
       with:
         path: /tmp/.buildx-cache
@@ -68,16 +68,8 @@ jobs:
 
     - name: Run Integration Tests
       id: run-int-tests
-      env:
-        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        GIT_SHA: ${{ github.sha }}
-        BUILD_NUMBER: ${{ github.run_number }}
       run: |
-        curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && chmod +x ./cc-test-reporter
-        ./cc-test-reporter before-build
-        mkdir $GITHUB_WORKSPACE/coverage
-        docker run --network ${{ job.container.network }} --rm -v $GITHUB_WORKSPACE/coverage:/app/coverage ${{ steps.build-image.outputs.image }} -e APP=fetch -e DEPLOY_ENV=travis -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_integration
-        ./cc-test-reporter after-build --exit-code 0 -t simplecov -p /app
+        docker run --network ${{ job.container.network }} --rm ${{ steps.build-image.outputs.image }} -e APP=fetch -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_integration
 
     - name: Docker Buildx (push)
       id: push-image-to-ecr
@@ -100,29 +92,33 @@ jobs:
           --tag $ECR_REGISTRY/$ECR_REPOSITORY:$BRANCH \
           --file ./Dockerfile ./
 
-    - name: Kick off Terraform deploy in sysops/ if not live
-      id: sysops-deploy
+    - name: Kick off Terraform QA deploy in sysops/
+      id: sysops-deploy-qa
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/main')
       run: |
-        BRANCH=$(echo "${GITHUB_REF#refs/*/}" | sed 's/.*\///')
-        if [[ "$BRANCH" != master ]]; then
-          curl \
+        curl --fail-with-body \
           -X POST \
           -H "Accept: application/vnd.github+json" \
           -H "Authorization: Bearer ${{ secrets.SYSOPS_RW_GITHUB_TOKEN }}" \
           -H "X-GitHub-Api-Version: 2022-11-28" \
           https://api.github.com/repos/meedan/sysops/actions/workflows/deploy_${{ github.event.repository.name }}.yml/dispatches \
-          -d '{"ref": "master", "inputs": {"git_sha": "${{ github.sha }}", "branch": "'$BRANCH'"}}'
-        fi
+          -d '{"ref": "master", "inputs": {"git_sha": "${{ github.sha }}", "type": "push"}}'
+
+    - name: Kick off Terraform Live deploy in sysops/
+      id: sysops-deploy-live
+      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+      run: |
+        RELEASE_TAG=$(echo "${GITHUB_REF#refs/*/}" | sed 's/.*\///')
+        curl --fail-with-body \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.SYSOPS_RW_GITHUB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/meedan/sysops/actions/workflows/deploy_${{ github.event.repository.name }}.yml/dispatches \
+          -d '{"ref": "master", "inputs": {"image_tag": "'$RELEASE_TAG'", "type": "tag"}}'
 
-    - name: Send Slack Notification
-      id: slack-notify
-      if: ${{ always() }}
-      uses: 8398a7/action-slack@v3
-      with:
-        status: ${{ job.status }}
-        author_name: 'Github Actions'
-        icon_emoji: ':octocat:'
-        fields: repo,message,author # selectable (default: repo,message)
-      env:
-        GITHUB_TOKEN: ${{ secrets.CONFIGURATOR_TOKEN }} # optional
-        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required
+    - name: Reset cache
+      id: reset-cache
+      if: ${{ success() || failure() }}
+      run: |
+        rm -rf /tmp/.buildx-cache

.github/workflows/ci-test-branches.yml

@@ -31,7 +31,7 @@ jobs:
       uses: docker/setup-buildx-action@v1
 
     - name: Cache Docker layers
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       id: cache
       with:
         path: /tmp/.buildx-cache
@@ -56,16 +56,8 @@ jobs:
 
     - name: Run Unit Tests
       id: run-unit-test
-      env:
-        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        GIT_SHA: ${{ github.sha }}
-        BUILD_NUMBER: ${{ github.run_number }}
       run: |
-        curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && chmod +x ./cc-test-reporter
-        ./cc-test-reporter before-build
-        mkdir $GITHUB_WORKSPACE/coverage
-        docker run --rm -v $GITHUB_WORKSPACE/coverage:/app/coverage ${{ steps.build-image.outputs.image }} -e APP=fetch -e DEPLOY_ENV=travis -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_unit
-        ./cc-test-reporter after-build --exit-code 0 -t simplecov -p /app
+        docker run --rm ${{ steps.build-image.outputs.image }} -e APP=fetch -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_unit
 
     - uses: 8398a7/action-slack@v3
       with:

.github/workflows/ci-test-pr.yml

@@ -39,7 +39,7 @@ jobs:
       uses: crazy-max/ghaction-docker-buildx@v3
 
     - name: Cache Docker layers
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       id: cache
       with:
         path: /tmp/.buildx-cache
@@ -64,23 +64,11 @@ jobs:
 
     - name: Run PR Tests
       id: run-tests
-      env:
-        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
-        GIT_SHA: ${{ github.sha }}
-        BUILD_NUMBER: ${{ github.run_number }}
       run: |
         if [[ "${{ github.event.pull_request.head.repo.fork }}" == "true" ]]; then
-          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && chmod +x ./cc-test-reporter
-          ./cc-test-reporter before-build
-          mkdir $GITHUB_WORKSPACE/coverage
-          docker run --rm -v $GITHUB_WORKSPACE/coverage:/app/coverage ${{ steps.build-image.outputs.image }} -e APP=fetch -e DEPLOY_ENV=travis -e GITHUB_TOKEN=none test_unit_fork
-          ./cc-test-reporter after-build --exit-code 0 -t simplecov -p /app
+          docker run --rm ${{ steps.build-image.outputs.image }} -e APP=fetch -e GITHUB_TOKEN=none test_unit_fork
         else
-          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter && chmod +x ./cc-test-reporter
-          ./cc-test-reporter before-build
-          mkdir $GITHUB_WORKSPACE/coverage
-          docker run --network ${{ job.container.network }} --rm -v $GITHUB_WORKSPACE/coverage:/app/coverage ${{ steps.build-image.outputs.image }} -e APP=fetch -e DEPLOY_ENV=travis -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_integration
-          ./cc-test-reporter after-build --exit-code 0 -t simplecov -p /app
+          docker run --network ${{ job.container.network }} --rm ${{ steps.build-image.outputs.image }} -e APP=fetch -e GITHUB_TOKEN=${{ secrets.CONFIGURATOR_TOKEN }} test_integration
         fi
 
     - uses: 8398a7/action-slack@v3

README.md

@@ -1,7 +1,4 @@
 ## Fetch
-[![Code Climate](https://api.codeclimate.com/v1/badges/42a4437feae3058176ff/maintainability)](https://codeclimate.com/repos/5ef4a2779226cb00dd00473b/maintainability)
-[![Test Coverage](https://api.codeclimate.com/v1/badges/42a4437feae3058176ff/test_coverage)](https://codeclimate.com/repos/5ef4a2779226cb00dd00473b/test_coverage)
-
 A Fact / Claim Review aggregation service.
 
 ## Development
@@ -62,13 +59,14 @@ class ParserName < ClaimReviewParser
   end
 
   (...)
-  
+
 end
 ```
+## Reimporting/Rebuilding/Verifying/Debugging from within a session
 
-## Reimporting/Rebuilding/Debugging an existing data Source
+### Reimporting/Rebuilding an existing data Source
 
-Sometimes, we need to rebuild a data source. One hypothetical reason could be if a publisher has added `ClaimReview` objects to all historical data, and we now want to re-capture high quality claims data. In that situation, once we have deployed our updates to the parser to collect those fixes, we will manually re-run a `RunClaimReviewParser` task in a console session with fetch - to run it within the session, we'd run:
+Sometimes, we need to rebuild a data source. One hypothetical reason could be if a publisher has added `ClaimReview` objects to all historical data, and we now want to re-capture high quality claims data. In that situation, once we have deployed our updates to the parser to collect those fixes, we will manually re-run a `RunClaimReviewParser` task in a console session with fetch:
 
 ```
 RunClaimReviewParser.new.perform("service_name", (Time.now-60*60*24*365*15).to_s, true)
@@ -81,7 +79,56 @@ RunClaimReviewParser.perform_async("service_name", (Time.now-60*60*24*365*15).to
 
 The way that all tasks run is that they paginate backwards in time until they see articles they've parsed before, *or*, if a date is specified, they keep paginating until they hit articles from that date. In this example, we set the date 15 years back to so that, in effect, we go re-build *all* data.
 
-If you just want to verify if/how an article is parsed by a service, you can run it this way:
+### Verifying imported claims' state
+
+After rebuilding the data source, you should check the claims' state. Run:
+
+```
+claim_reviews = API.claim_reviews(service: "service_name", per_page: 1000);false
+```
+
+After that you can go through the claims and look for specific urls, dates, anything that might help confirm their state is as expected.
+
+### Verifying services' state
+If you want to confirm which services are currently running (not deprecated). Run:
+
+```
+API.services
+```
+
+Which returns something like this:
+
+```
+{:services=>[{:service=>"service_name", :count=>1000, :earliest=>"2000-01-01", :latest=>"2025-09-08"}, {:service=>"another_service_name", :count=>1000, :earliest=>"2005-01-01", :latest=>"2025-09-08"}]}
+```
+
+### Checking services' jobs
+We use Sidekiq to run our background jobs.
+
+You can check if the job for a specific service is enqueued:
+```
+RunClaimReviewParser.not_enqueued_anywhere_else("service_name")
+```
+
+But you can also manually check sets and queues to look for jobs:
+```
+[Sidekiq::RetrySet, Sidekiq::ScheduledSet, Sidekiq::Queue].each do |queue|
+  queue.new.each { |job| p job }
+end
+```
+
+### Managing services' subscriptions
+A service might have many subscriptions. We might want to keep the service, but deprecate an specific subscription. Or, we might want to add a new subscription. Here are some usefull commands:
+
+```
+API.get_subscriptions({service: "service_name"})
+API.add_subscriptions({service: "service_name", url: "webhook_url", language: "language"})
+API.remove_subscriptions({service: "service_name", url: "webhook_url"})
+```
+
+### Debugging
+If you just want to verify if/how an article is parsed by a service, run:
+
 ```
 ServiceParserClassHere.test_parser_on_url(url)
 ```

lib/claim_review_parsers/alter_midya.rb

@@ -4,6 +4,9 @@
 # Parser for https://www.altermidya.net/category/factsfirstph/page/2/
 class AlterMidya < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
 
   def hostname
     'https://www.altermidya.net'

lib/claim_review_parsers/aos_fatos.rb

@@ -3,6 +3,10 @@
 # Parser for https://www.aosfatos.org/noticias/checamos/
 class AosFatos < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def hostname
     'https://www.aosfatos.org'
   end

lib/claim_review_parsers/climate_fact_check.rb

@@ -4,6 +4,10 @@
 # Parser for https://climatefactchecks.org
 class ClimateFactCheck < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def hostnames
     [
       "https://climatefactchecks.org/category/fact-check/",

lib/claim_review_parsers/desi_facts.rb

@@ -3,6 +3,9 @@
 # Parser for https://www.desifacts.org/
 class DesiFacts < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
 
   def self.includes_service_keyword
     true

lib/claim_review_parsers/gesis_claims.rb

@@ -3,6 +3,10 @@
 # Parser for
 class GESISClaims < ClaimReviewParser
   include GenericRawClaimParser
+  def self.deprecated
+    true
+  end
+
   def post_request_get_fact_ids(page, limit)
     RestClient::Request.execute(
       :method => :post,

lib/claim_review_parsers/google_fact_check.rb

@@ -1,6 +1,10 @@
 # frozen_string_literal: true
 
 class GoogleFactCheck < ClaimReviewParser
+  def self.deprecated
+    true
+  end
+
   def host
     'https://factchecktools.googleapis.com'
   end

lib/claim_review_parsers/mindanao_gold_star_daily.rb

@@ -4,6 +4,9 @@
 # Parser for https://mindanaogoldstardaily.com/category/factsfirstph-2/
 class MindanaoGoldStarDaily < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
 
   def hostname
     'https://mindanaogoldstardaily.com'

lib/claim_review_parsers/reuters.rb

@@ -41,7 +41,7 @@ def score_map
   end
 
   def claim_result_from_subhead(page)
-    header = page.search('div[class^="article-body__content"]').first.search('h2[data-testid="Heading"]').last
+    header = page.search('div[class^="article-body-module__content"]').first.search('h2[data-testid="Heading"]').last
     if header
       header.next_sibling.text.split(".").first
     end
@@ -63,7 +63,7 @@ def parse_raw_claim_review(raw_claim_review)
       author: raw_claim_review.dig('raw_response', 'authors', 0, 'name'),
       author_link: nil,
       claim_review_headline: raw_claim_review.dig('raw_response', 'title'),
-      claim_review_body: raw_claim_review['page'].search('div[class^="article-body__content"]').children.text,
+      claim_review_body: raw_claim_review['page'].search('div[class^="article-body-module__content"]').children.text,
       claim_review_image_url: raw_claim_review.dig('raw_response', 'thumbnail', 'url'),
       claim_review_result: claim_result,
       claim_review_result_score: score_map[claim_result],

lib/claim_review_parsers/tayo.rb

@@ -3,6 +3,10 @@
 # Parser for https://www.desifacts.org/
 class Tayo < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def self.includes_service_keyword
     true
   end

lib/claim_review_parsers/telemundo.rb

@@ -3,6 +3,10 @@
 # Parser for https://www.telemundo.com
 class Telemundo < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def initialize(cursor_back_to_date = nil, overwrite_existing_claims=false, send_notifications = true)
     cursor_back_to_date ||= Time.now-60*60*24*7 #looks like their pagination just loops back to other stories, breaking our iteration completion logic. Override to short window when not specified.
     super(cursor_back_to_date, overwrite_existing_claims, send_notifications)

lib/claim_review_parsers/tfc_taiwan.rb

@@ -3,6 +3,10 @@
 # Parser for https://tfc-taiwan.org.tw
 class TFCTaiwan < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def hostname
     'https://tfc-taiwan.org.tw'
   end

lib/claim_review_parsers/tsek.rb

@@ -2,6 +2,10 @@
 
 class Tsek < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def initialize(cursor_back_to_date = nil, overwrite_existing_claims=false, send_notifications = true)
     super(cursor_back_to_date, overwrite_existing_claims, send_notifications)
     @fact_list_page_parser = 'json'
@@ -98,4 +102,4 @@ def parse_raw_claim_review(raw_claim_review)
       raw_claim_review: news_article
     }
   end
-end
+end

lib/claim_review_parsers/univision.rb

@@ -3,6 +3,10 @@
 # Parser for https://syndicator.univision.com
 class Univision < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def initialize(cursor_back_to_date = nil, overwrite_existing_claims=false, send_notifications = true)
     super(cursor_back_to_date, overwrite_existing_claims, send_notifications)
     @fact_list_page_parser = 'json'

lib/claim_review_parsers/washington_post.rb

@@ -2,6 +2,10 @@
 
 class WashingtonPost < ClaimReviewParser
   include PaginatedReviewClaims
+  def self.deprecated
+    true
+  end
+
   def initialize(cursor_back_to_date = nil, overwrite_existing_claims=false, send_notifications = true)
     super(cursor_back_to_date, overwrite_existing_claims, send_notifications)
     @fact_list_page_parser = 'json'

pull_request_template.md

@@ -17,6 +17,7 @@ Please describe parts of the change that require extra attention during code rev
 
 ## Checklist
 
+- [ ] I performed a rebase of my changes instead of a merge where possible.
 - [ ] I have performed a self-review of my own code
 - [ ] I have added unit and feature tests, if the PR implements a new feature or otherwise would benefit from additional testing
 - [ ] I have added regression tests, if the PR fixes a bug