Separate consumers in catalog service

.env.example

@@ -86,4 +86,6 @@ MONGO_DATABASE_URL=mongodb://mongo:27017
 #MONGO_DATABASE_URL=mongodb://mongo1:27017,mongo2:27018/?replicaSet=my-replica-set&readPreference=secondary
 
 # Catalog Service Env
+LANGAUGE_LIST=""
 ELASTIC_SEARCH_URL=http://elasticsearch:9200
+

docker-compose-for-local.yaml

@@ -206,26 +206,51 @@ services:
     ports:
       - 9200:9200
       - 9300:9300
-  catalog-service:
+  catalog-service-transformer:
     restart: always
     build:
       context: catalog-service
-      dockerfile: Dockerfile
+      dockerfile: dockerfiles/transformerDockerfile
     depends_on:
-      - elasticsearch
       - rabbitmq
-      - mongo
+      - mongo1
     environment:
-      ENV: "dev"
+      ENV: dev
       MONGO_DATABASE_URL: ${MONGO_DATABASE_URL}
-      RABBITMQ_HOST: ${RABBITMQ_HOST}
       ELASTIC_SEARCH_URL: ${ELASTIC_SEARCH_URL}
-      BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY}
+      RABBITMQ_HOST: "rabbitmq"
+      CONSUMER_MAX_WORKERS: "3"
+      LANGUAGE_LIST: ${LANGUAGE_LIST}
+  catalog-service-es-dumper:
+    restart: always
+    build:
+      context: catalog-service
+      dockerfile: dockerfiles/esDumperDockerfile
+    container_name: catalog-service-es-dumper
+    depends_on:
+      - elasticsearch
+      - rabbitmq
+    environment:
+      ENV: dev
+      RABBITMQ_HOST: "rabbitmq"
+      ELASTIC_SEARCH_URL: ${ELASTIC_SEARCH_URL}
+      CONSUMER_MAX_WORKERS: "20"
+  catalog-service-translator:
+    restart: always
+    build:
+      context: catalog-service
+      dockerfile: dockerfiles/translatorDockerfile
+    depends_on:
+      - rabbitmq
+      - cache
+    environment:
+      ENV: dev
+      RABBITMQ_HOST: "rabbitmq"
+      REDIS_HOST: "cache"
+      CONSUMER_MAX_WORKERS: "1"
       BHASHINI_USERID: ${BHASHINI_USERID}
-      REDIS_HOST: cache
-      LANGUAGE_LIST: ""
-      CONSUMER_MAX_WORKERS: "5"
-      BULK_CHUNK_SIZE: "5000"
+      BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY}
+      LANGUAGE_LIST: ${LANGUAGE_LIST}
 
   # Ancillary Services
   mmi-client:

docker-compose.yaml

@@ -228,32 +228,69 @@ services:
     ports:
       - 9200:9200
       - 9300:9300
-  catalog-service:
+  catalog-service-transformer:
     restart: always
-    mem_limit: 5gb
+    mem_limit: 4gb
     logging:
       driver: "json-file"
       options:
         max-size: "100m"
         max-file: "3"
     build:
       context: catalog-service
-      dockerfile: Dockerfile
+      dockerfile: dockerfiles/transformerDockerfile
     depends_on:
-      - elasticsearch
       - rabbitmq
-      - mongo
+      - mongo1
     environment:
-      ENV: "dev"
+      ENV: dev
       MONGO_DATABASE_URL: ${MONGO_DATABASE_URL}
-      RABBITMQ_HOST: ${RABBITMQ_HOST}
       ELASTIC_SEARCH_URL: ${ELASTIC_SEARCH_URL}
-      BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY}
+      RABBITMQ_HOST: "rabbitmq"
+      CONSUMER_MAX_WORKERS: "3"
+      LANGUAGE_LIST: ${LANGUAGE_LIST}
+  catalog-service-es-dumper:
+    restart: always
+    mem_limit: 3gb
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "3"
+    build:
+      context: catalog-service
+      dockerfile: dockerfiles/esDumperDockerfile
+    container_name: catalog-service-es-dumper
+    depends_on:
+      - elasticsearch
+      - rabbitmq
+    environment:
+      ENV: dev
+      RABBITMQ_HOST: "rabbitmq"
+      ELASTIC_SEARCH_URL: ${ELASTIC_SEARCH_URL}
+      CONSUMER_MAX_WORKERS: "20"
+  catalog-service-translator:
+    restart: always
+    mem_limit: 5gb
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "100m"
+        max-file: "10"
+    build:
+      context: catalog-service
+      dockerfile: dockerfiles/translatorDockerfile
+    depends_on:
+      - rabbitmq
+      - cache
+    environment:
+      ENV: dev
+      RABBITMQ_HOST: "rabbitmq"
+      REDIS_HOST: "cache"
+      CONSUMER_MAX_WORKERS: "1"
       BHASHINI_USERID: ${BHASHINI_USERID}
-      REDIS_HOST: cache
-      LANGUAGE_LIST: ""
-      CONSUMER_MAX_WORKERS: "5"
-      BULK_CHUNK_SIZE: "5000"
+      BHASHINI_ULCA_API_KEY: ${BHASHINI_ULCA_API_KEY}
+      LANGUAGE_LIST: ${LANGUAGE_LIST}
   cache:
     image: redis:6.2-alpine
     restart: always
@@ -263,7 +300,6 @@ services:
       - 6379
     command: redis-server --save 20 1 --loglevel warning
 
-
   # Ancillary Services
   mmi-client:
     image: navdeep710/ondc-ancillary-services:v3
@@ -299,6 +335,3 @@ services:
     image: shameerkc/bugzilla-dev-sqlite
     ports:
       - "90:80"
-
-
-