From 93c1490b437516d3b4d76ad72d4c9ed0c8c12a9a Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 20 Sep 2023 19:57:15 +0530 Subject: [PATCH 01/88] Dev #104439 - Create new Flink job to validate the program certificate event generation --- credential-generator/pom.xml | 1 + .../src/main/resources/log4j.properties | 11 +++ .../resources/program-cert-pre-processor.conf | 38 +++++++++ .../job/programcert/domain/Event.scala | 25 ++++++ .../job/programcert/domain/Models.scala | 26 ++++++ .../functions/IssueCertificateHelper.scala | 17 ++++ .../functions/ProgramCertPreProcessorFn.scala | 57 +++++++++++++ .../task/ProgramCertPreProcessorConfig.scala | 85 +++++++++++++++++++ .../task/ProgramCertPreProcessorTask.scala | 58 +++++++++++++ .../src/test/resources/logback-test.xml | 16 ++++ .../src/test/resources/test.conf | 45 ++++++++++ .../src/test/resources/test.cql | 85 +++++++++++++++++++ .../programcert/fixture/EventFixture.scala | 15 ++++ .../ProgramCertPreProcessFnTestSpec.scala | 6 ++ .../ProgramCertPreProcessorTaskSpec.scala | 9 ++ 15 files changed, 494 insertions(+) create mode 100644 credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties create mode 100644 credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala create mode 100644 credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala create mode 100644 credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml create mode 100644 credential-generator/program-cert-pre-processor/src/test/resources/test.conf create mode 100644 credential-generator/program-cert-pre-processor/src/test/resources/test.cql create mode 100644 credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala create mode 100644 credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala create mode 100644 credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala diff --git a/credential-generator/pom.xml b/credential-generator/pom.xml index 2c7fadf4a..a18fef683 100644 --- a/credential-generator/pom.xml +++ b/credential-generator/pom.xml @@ -14,6 +14,7 @@ collection-cert-pre-processor certificate-processor collection-certificate-generator + program-cert-pre-processor diff --git a/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties b/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties new file mode 100644 index 000000000..cacd49c79 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties @@ -0,0 +1,11 @@ +# log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.file=course-metrics-updater.log +log4j.appender.file.append=true +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.MaxFileSize=256KB +log4j.appender.file.MaxBackupIndex=4 +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf new file mode 100644 index 000000000..50cff9b9d --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf @@ -0,0 +1,38 @@ +include "base-config.conf" + +kafka { + input.topic = "sunbirddev.program.cert.request" + output.topic = "sunbirddev.program.cert.output.request" + output.failed.topic = "sunbirddev.issue.certificate.failed" + groupId = "collection-cert-pre-processor-group" +} + +task { + consumer.parallelism = 1 + parallelism = 1 + generate_certificate.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + user_enrolments.table = "user_enrolments" + course_batch.table = "course_batch" + assessment_aggregator.table = "assessment_aggregator" + user_activity_agg.table = "user_activity_agg" +} + +cert_domain_url="https://dev.sunbirded.org" +user_read_api = "/private/user/v1/read" +content_read_api = "/content/v3/read" + +service { + content.basePath = "http://localhost:9000" + learner.basePath = "http://localhost:9000" +} + +redis-meta { + host = localhost + port = 6379 +} +assessment.metrics.supported.contenttype = ["SelfAssess"] +enable.suppress.exception = true diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala new file mode 100644 index 000000000..2550d77a1 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala @@ -0,0 +1,25 @@ +package org.sunbird.job.programcert.domain + +import org.sunbird.job.domain.reader.JobRequest +import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig + +class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { + + def action:String = readOrDefault[String]("edata.action", "") + + def batchId: String = readOrDefault[String]("edata.batchId", "") + + def courseId: String = readOrDefault[String]("edata.courseId", "") + + def userId: String = { + val list = readOrDefault[List[String]]("edata.userIds", List[String]()) + if(list.isEmpty) "" else list.head + } + + def eData: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("edata", Map[String, AnyRef]()) + + def isValid()(config: ProgramCertPreProcessorConfig): Boolean = { + config.programCertProgram.equalsIgnoreCase(action) && !batchId.isEmpty && !courseId.isEmpty && + !userId.isEmpty + } +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala new file mode 100644 index 000000000..635f881d9 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala @@ -0,0 +1,26 @@ +package org.sunbird.job.programcert.domain + +import java.util.{Date, UUID} + +case class EnrolledUser(userId: String, oldId: String = null, issuedOn: Date = null, additionalProps: Map[String, Any] = Map[String, Any]()) + +case class AssessedUser(userId: String, additionalProps: Map[String, Any] = Map[String, Any]()) + + +case class ActorObject(id: String = "Certificate Generator", `type`: String = "System") + +case class EventContext(pdata: Map[String, String] = Map("ver" -> "1.0", "id" -> "org.sunbird.learning.platform")) + + +case class EventObject(id: String, `type`: String = "GenerateCertificate") + +case class BEJobRequestEvent(actor: ActorObject= ActorObject(), + eid: String = "BE_JOB_REQUEST", + edata: Map[String, AnyRef], + ets: Long = System.currentTimeMillis(), + context: EventContext = EventContext(), + mid: String = s"LMS.${UUID.randomUUID().toString}", + `object`: EventObject + ) + +case class AssessmentUserAttempt(contentId: String, score: Double, totalScore: Double) \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala new file mode 100644 index 000000000..9bd865288 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala @@ -0,0 +1,17 @@ +package org.sunbird.job.programcert.functions + +import java.text.SimpleDateFormat +import com.datastax.driver.core.querybuilder.QueryBuilder +import com.datastax.driver.core.{Row, TypeTokens} +import org.apache.commons.lang3.StringUtils +import org.slf4j.LoggerFactory +import org.sunbird.job.Metrics +import org.sunbird.job.cache.DataCache +import org.sunbird.job.programcert.domain.{AssessedUser, AssessmentUserAttempt, EnrolledUser, Event} +import org.sunbird.job.programcert.task.{ProgramCertPreProcessorConfig} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} + +import scala.collection.JavaConverters._ + +trait IssueCertificateHelper { +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala new file mode 100644 index 000000000..75c004fe2 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -0,0 +1,57 @@ +package org.sunbird.job.programcert.functions + +import com.datastax.driver.core.querybuilder.QueryBuilder +import com.datastax.driver.core.{Row, TypeTokens} +import com.google.common.reflect.TypeToken +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.KeyedProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.InvalidEventException +import org.sunbird.job.programcert.domain.Event +import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} + +import scala.collection.JavaConverters._ + +class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil) + (implicit val stringTypeInfo: TypeInformation[String], + @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessKeyedFunction[String, Event, String](config) with IssueCertificateHelper { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) + private var cache: DataCache = _ + private var contentCache: DataCache = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + val redisConnect = new RedisConnect(config) + cache = new DataCache(config, redisConnect, config.collectionCacheStore, List()) + cache.init() + + /*val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) + contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) + contentCache.init()*/ + logger.info("This is the flink testing") + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + override def metricsList(): List[String] = { + List(config.totalEventsCount, config.dbReadCount, config.dbUpdateCount, config.failedEventCount, config.skippedEventCount, config.successEventCount, + config.cacheHitCount) + } + + override def processElement(event: Event, + context: KeyedProcessFunction[String, Event, String]#Context, + metrics: Metrics): Unit = { + logger.info("Inside the Cert Processor for Program Event"); + } +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala new file mode 100644 index 000000000..bffb7bb22 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -0,0 +1,85 @@ +package org.sunbird.job.programcert.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.job.BaseJobConfig + +import java.util + +class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJobConfig(config, "collection-cert-pre-processor") { + + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + + //Redis config + val collectionCacheStore: Int = 0 + val contentCacheStore: Int = 5 + val metaRedisHost: String = config.getString("redis-meta.host") + val metaRedisPort: Int = config.getInt("redis-meta.port") + + //kafka config + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaOutputTopic: String = config.getString("kafka.output.topic") + val certificatePreProcessorConsumer: String = "collection-cert-pre-processor-consumer" + val generateCertificateProducer = "generate-certificate-sink" + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val generateCertificateParallelism:Int = config.getInt("task.generate_certificate.parallelism") + + //Tags + val generateCertificateOutputTagName = "generate-certificate-request" + val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) + + //Cassandra config + val dbHost: String = config.getString("lms-cassandra.host") + /*val dbHost: String = "192.168.56.101" + val dbPort: Int = 9042*/ + val dbPort: Int = config.getInt("lms-cassandra.port") + val keyspace: String = config.getString("lms-cassandra.keyspace") + val courseTable: String = config.getString("lms-cassandra.course_batch.table") + val userEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") + val assessmentTable: String = config.getString("lms-cassandra.assessment_aggregator.table") + val useActivityAggTable: String = config.getString("lms-cassandra.user_activity_agg.table") + val dbBatchId = "batchid" + val dbCourseId = "courseid" + val dbUserId = "userid" + + //API URL + val contentBasePath = config.getString("service.content.basePath") + val learnerBasePath = config.getString("service.learner.basePath") + val userReadApi = config.getString("user_read_api") + val contentReadApi = config.getString("content_read_api") + + // Metric List + val totalEventsCount = "total-events-count" + val successEventCount = "success-events-count" + val failedEventCount = "failed-events-count" + val skippedEventCount = "skipped-event-count" + val dbReadCount = "db-read-count" + val dbUpdateCount = "db-update-count" + val cacheHitCount = "cache-hit-cout" + + //Constants + val issueCertificate = "issue-certificate" + val certTemplates = "cert_templates" + val criteria: String = "criteria" + val enrollment: String = "enrollment" + val assessment: String = "assessment" + val users: String = "users" + val active: String = "active" + val issuedCertificates: String = "issued_certificates" + val status: String = "status" + val name: String = "name" + val user: String= "user" + val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") + val identifier: String = "identifier" + val completedOn: String = "completedon" + val issuer: String = "issuer" + val signatoryList: String = "signatoryList" + val certBasePath: String = config.getString("cert_domain_url") + "/certs" + val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") + val userAccBlockedErrCode = "UOS_USRRED0006" + val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false + val programCertProgram: String = "program_cert_program" + +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala new file mode 100644 index 000000000..41a7bf873 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala @@ -0,0 +1,58 @@ +package org.sunbird.job.programcert.task + +import java.io.File +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.slf4j.LoggerFactory +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.programcert.domain.Event +import org.sunbird.job.programcert.functions.ProgramCertPreProcessorFn +import org.sunbird.job.util.{FlinkUtil, HttpUtil} + +class ProgramCertPreProcessorTask(config: ProgramCertPreProcessorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorTask]) + + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) + logger.info("This is under process for task") + val progressStream = + env.addSource(source).name(config.certificatePreProcessorConsumer) + .uid(config.certificatePreProcessorConsumer).setParallelism(config.kafkaConsumerParallelism) + .rebalance + .keyBy(new ProgramCertPreProcessorKeySelector()) + .process(new ProgramCertPreProcessorFn(config, httpUtil)) + .name("program-cert-pre-processor").uid("program-cert-pre-processor") + .setParallelism(config.parallelism) + + progressStream.getSideOutput(config.generateCertificateOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaOutputTopic)) + .name(config.generateCertificateProducer).uid(config.generateCertificateProducer).setParallelism(config.generateCertificateParallelism) + env.execute(config.jobName) + } +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster + +object ProgramCertPreProcessorTask { + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("program-cert-pre-processor.conf").withFallback(ConfigFactory.systemEnvironment())) + val certificatePreProcessorConfig = new ProgramCertPreProcessorConfig(config) + val kafkaUtil = new FlinkKafkaConnector(certificatePreProcessorConfig) + val httpUtil = new HttpUtil() + val task = new ProgramCertPreProcessorTask(certificatePreProcessorConfig, kafkaUtil, httpUtil) + task.process() + } +} + +class ProgramCertPreProcessorKeySelector extends KeySelector[Event, String] { + override def getKey(event: Event): String = Set(event.userId, event.courseId, event.batchId).mkString("_") +} \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml b/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml new file mode 100644 index 000000000..e81294323 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/test.conf b/credential-generator/program-cert-pre-processor/src/test/resources/test.conf new file mode 100644 index 000000000..efad514b6 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/test.conf @@ -0,0 +1,45 @@ +include "base-test.conf" + +kafka { + input.topic = "flink.issue.certificate.request" + output.topic = "flink.generate.certificate.request" + output.failed.topic = "flink.issue.certificate.failed" + groupId = "flink-collection-cert-pre-processor-group" +} + +task { + consumer.parallelism = 1 + parallelism = 1 + generate_certificate.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + user_enrolments.table = "user_enrolments" + course_batch.table = "course_batch" + assessment_aggregator.table = "assessment_aggregator" + user_activity_agg.table = "user_activity_agg" + host = "localhost" + port = "9142" +} + +dp-redis { + host = localhost + port = 6340 + database.index = 5 +} + +cert_domain_url="https://dev.sunbirded.org" +user_read_api = "/private/user/v1/read" +content_read_api = "/content/v3/read" + +service { + content.basePath = "http://localhost:9000/content" + learner.basePath = "http://localhost:9000/learner" +} + +redis-meta { + host = localhost + port = 6379 +} +assessment.metrics.supported.contenttype = ["selfAssess"] diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/test.cql b/credential-generator/program-cert-pre-processor/src/test/resources/test.cql new file mode 100644 index 000000000..5a5b056cf --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/test.cql @@ -0,0 +1,85 @@ +CREATE KEYSPACE IF NOT EXISTS sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE IF NOT EXISTS sunbird_courses.course_batch ( + courseid text, + batchid text, + cert_templates map>>, + createdby text, + createddate text, + createdfor list, + description text, + enddate text, + enrollmentenddate text, + enrollmenttype text, + mentors list, + name text, + startdate text, + status int, + updateddate text, + PRIMARY KEY (courseid, batchid) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_enrolments ( + userid text, + courseid text, + batchid text, + active boolean, + addedby text, + certificates list>>, + completedon timestamp, + completionpercentage int, + contentstatus map, + datetime timestamp, + issued_certificates list>>, + enrolleddate text, + lastreadcontentid text, + lastreadcontentstatus int, + progress int, + status int, + PRIMARY KEY (userid, courseid, batchid) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator ( + user_id text, + course_id text, + batch_id text, + content_id text, + attempt_id text, + created_on timestamp, + grand_total text, + last_attempted_on timestamp, + total_max_score double, + total_score double, + updated_on timestamp, + PRIMARY KEY ((user_id, course_id), batch_id, content_id, attempt_id) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg ( + activity_id text, + user_id text, + activity_type text, + context_id text, + agg Map, + aggregates Map, + agg_last_updated Map, + PRIMARY KEY ((activity_type, activity_id), context_id, user_id) +); + + +//event 1 +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('do_11309999837886054415','0131000245281587206',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}, "assessment": {"score": {">=": 80}}}', 'identifier': 'template_01_dev_001', 'url': 'template-url.svg', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course Completion Certificate', 'notifyTemplate': '{"subject":"Completion certificate","stateImgUrl":"https://sunbirddev.blob.core.windows.net/orgemailtemplate/img/File-0128212938260643843.png","regardsperson":"Chairperson","regards":"Minister of Gujarat","emailTemplateType":"defaultCertTemp"}', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, issued_certificates, completedon, status, active) VALUES ('user001','do_11309999837886054415','0131000245281587206',[{'identifier': 'certificateId', 'lastIssuedOn': '2019-08-21', 'name': 'Course Completion Certificate', 'token': 'P4L3Y9'}], toTimeStamp(toDate(now())), 2, true); +INSERT INTO sunbird_courses.assessment_aggregator(user_id, course_id, batch_id, content_id, attempt_id, total_max_score, total_score) VALUES ('user001','do_11309999837886054415','0131000245281587206', 'content_001', 'attempt_001', 1, 1); +INSERT INTO sunbird_courses.user_activity_agg(activity_id, user_id, activity_type, context_id, aggregates) VALUES ('do_11309999837886054415', 'user001', 'Course', 'cb:0131000245281587206', {'score:content_001': 1, 'max_score:content_001': 1}); +//event 2 +INSERT INTO sunbird_courses.course_batch(courseid, batchid,cert_templates) VALUES ('do_11309999837886054416','0131000245281587207',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}}','url': 'template-url.svg', 'identifier': 'template_01_dev_001', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course merit certificate', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid,completedon) VALUES ('user002','do_11309999837886054416','0131000245281587207', toTimeStamp(toDate(now()))); +//empty cert_template +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('course_002','batch_002',{}); + +//event 3 +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('course_003','batch_003',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}, "assessment": {"score": {">=": 80}}}', 'identifier': 'template_01_dev_001', 'url': 'template-url.svg', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course merit certificate', 'notifyTemplate': '{"subject":"Completion certificate","stateImgUrl":"https://sunbirddev.blob.core.windows.net/orgemailtemplate/img/File-0128212938260643843.png","regardsperson":"Chairperson","regards":"Minister of Gujarat","emailTemplateType":"defaultCertTemp"}', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +//user with empty issued certificate +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, issued_certificates, completedon, status, active) VALUES ('user003','course_003','batch_003',[], toTimeStamp(toDate(now())), 2, true); +INSERT INTO sunbird_courses.assessment_aggregator(user_id, course_id, batch_id, content_id, attempt_id, total_max_score, total_score) VALUES ('user003','course_003','batch_003', 'content_001', 'attempt_001', 1, 1); +INSERT INTO sunbird_courses.user_activity_agg(activity_id, user_id, activity_type, context_id, agg) VALUES ('course_003', 'user003', 'Course', 'cb:batch_003', {'score:content_001': 1, 'max_score:content_001': 1}); diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala new file mode 100644 index 000000000..945ddd9e6 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala @@ -0,0 +1,15 @@ +package org.sunbird.job.programcert.fixture + +object EventFixture { + + val EVENT_1: String = """{"eid": "BE_JOB_REQUEST","ets": 1621833310477,"mid": "LP.1621833310477.429e795f-6d3e-4a11-8754-685984d62d10","actor": {"id": "Course Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "0131000245281587206_do_11309999837886054416","type": "CourseCertificateGeneration"},"edata": {"userIds": ["user001"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "0131000245281587206","reIssue": true,"courseId": "do_11309999837886054415"}}""" + val USER_1: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"user","lastName":"name","rootOrgId": "Org001","userLocations":[{"code":"29","name":"Karnataka","id":"027f81d8-0a2c-4fc6-96ac-59fe4cea3abf","type":"state","parentId":null},{"code":"2920","name":"BENGALURU URBAN SOUTH","id":"fa17379e-f8d5-4403-ae64-9e339f1dd599","type":"district","parentId":"027f81d8-0a2c-4fc6-96ac-59fe4cea3abf"}],"rootOrg":{"keys":{}}}}}""" + val CONTENT_1: String = """{"id":"api.content.read","ver":"3.0","ts":"2021-05-27T10:31:33ZZ","params":{"resmsgid":"316b05dd-df1a-4867-968a-042bb06a710f","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"objectType":"Content","primaryCategory":"Course","contentType":"Course","identifier":"do_11309999837886054416","languageCode":["en"],"name":"test Course"}}}""" + val TEMPLATE_1: String = """{"criteria":"{\"enrollment\":{\"status\":2},\"assessment\":{\"score\":{\">=\":50}}}","identifier":"template_svg_04-prad","issuer":"{\"name\":\"Gujarat Council of Educational Research and Training\",\"url\":\"https://gcert.gujarat.gov.in/gcert/\"}","name":"Course Completion Certificate","signatoryList":"[{\"image\":\"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg\",\"name\":\"CEO Gujarat\",\"id\":\"CEO\",\"designation\":\"CEO\"}]","url":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/template_svg_04-prad/artifact/template-1.svg","additionalProps": "{\"enrollment\":[\"completedOn\"],\"location\":[\"state\",\"district\",\"school\"],\"assessment\":[\"score\"],\"course\":[\"name\"]}"}""" + + val USER_2_EMPTY_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Rajesh","lastName":"","rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + val USER_3_NULL_VALUE_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Suresh","lastName":null,"rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + val USER_4_NULL_STRING_VALUE_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Manju","lastName":"null","rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + + +} diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala new file mode 100644 index 000000000..4054696fe --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala @@ -0,0 +1,6 @@ +package org.sunbird.job.programcert.function.spec + +import org.sunbird.spec.BaseTestSpec + +class ProgramCertPreProcessFnTestSpec extends BaseTestSpec { +} diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala new file mode 100644 index 000000000..03077a9d8 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala @@ -0,0 +1,9 @@ +package org.sunbird.job.programcert.function.spec + +import org.scalatest.DoNotDiscover +import org.sunbird.spec.BaseTestSpec + +@DoNotDiscover +class ProgramCertPreProcessorTaskSpec extends BaseTestSpec { + +} \ No newline at end of file From 9b49a3b9c7a9ce20183fde4d1a17b83c132da7de Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 21 Sep 2023 01:10:27 +0530 Subject: [PATCH 02/88] Dev #104439 - Create new Flink job to validate the program certificate event generation --- .../functions/IssueCertificateHelper.scala | 27 +++++++ .../functions/ProgramCertPreProcessorFn.scala | 72 ++++++++++++++++++- .../task/ProgramCertPreProcessorConfig.scala | 6 ++ 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala index 9bd865288..1c3eb4bd9 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala @@ -14,4 +14,31 @@ import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} import scala.collection.JavaConverters._ trait IssueCertificateHelper { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) + def getAPICall(url: String, responseParam: String)(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil, metrics: Metrics): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil.deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] + } else if (400 == response.status && response.body.contains(config.userAccBlockedErrCode)) { + metrics.incCounter(config.skippedEventCount) + logger.error(s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body) + Map[String, AnyRef]() + } else { + throw new Exception(s"Error from get API : ${url}, with response: ${response}") + } + } + + def getCourseReadDetails(courseId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): List[String] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=primaryCategory,identifier,parentCollections" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + response.getOrElse(config.parentCollections, List()).asInstanceOf[List[String]] + } else { + courseMetadata.getOrElse(config.parentCollections, List()).asInstanceOf[List[String]] + } + } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 75c004fe2..7c9ce8a1b 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -15,6 +15,8 @@ import org.sunbird.job.util.{CassandraUtil, HttpUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} +import scala.util.control.Breaks.{break, breakable} class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil) (implicit val stringTypeInfo: TypeInformation[String], @@ -32,9 +34,9 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: cache = new DataCache(config, redisConnect, config.collectionCacheStore, List()) cache.init() - /*val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) + val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) - contentCache.init()*/ + contentCache.init() logger.info("This is the flink testing") } @@ -52,6 +54,70 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: override def processElement(event: Event, context: KeyedProcessFunction[String, Event, String]#Context, metrics: Metrics): Unit = { - logger.info("Inside the Cert Processor for Program Event"); + try { + val getParentForCourse = getCourseReadDetails(event.courseId)(metrics, config, contentCache, httpUtil).asInstanceOf[List[String]] + if(!getParentForCourse.isEmpty) { + for (parentCourse <- getParentForCourse) { + val childrenForCuratedProgram = getProgramChildren(parentCourse)(metrics, config, contentCache, httpUtil).asInstanceOf[Map[String, AnyRef]] + if (!childrenForCuratedProgram.isEmpty) { + val contentDataForProgram = childrenForCuratedProgram.get(config.childrens).asInstanceOf[List[Map[String, AnyRef]]] + var isProgramCertificateToBeGenerated: Boolean = true; + for (childNode <- contentDataForProgram) { + val courseId: String = childNode.get(config.identifier).asInstanceOf[String] + val batchesForCourse: java.util.List[java.util.Map[String, AnyRef]] = childNode.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + val filteredBatches = batchesForCourse.filter(batch => batch.get("status") != 2).toList + val batchId: String = filteredBatches.get(0).get("batchId").asInstanceOf[String] + val userId: String = event.userId + val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbBatchId.toLowerCase -> batchId, config.dbCourseId.toLowerCase -> courseId) + val isCertificateIssued: List[Row] = getCourseIssuedCertificateForUser(primaryFields)(metrics).asInstanceOf[List[Row]] + breakable { + if (isCertificateIssued == null) { + isProgramCertificateToBeGenerated = false; + break + } + } + } + if(isProgramCertificateToBeGenerated) { + //Need to add kafka event to generate Certificate + } + } + } + } + } catch { + case ex: Exception => { + throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) + } + } + logger.info("Inside the Process ElementForProgram"); } + + def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): Map[String, AnyRef] = { + val query = QueryBuilder.select().all().from(config.contentHierarchyKeySpace, config.contentHierarchyTable) + .where(QueryBuilder.eq(config.identifier, programId)) + val row: Row = cassandraUtil.findOne(query.toString) + if (null != row) { + val templates = row.getMap(config.Hierarchy, TypeToken.of(classOf[String]), TypeTokens.mapOf(classOf[String], classOf[String])) + templates.asScala.map(template => (template._1 -> template._2.asScala.toMap)).toMap + } else { + Map[String, Map[String, String]]() + } + } + + private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics) = { + logger.info("primary columns {}", columns) + val selectWhere = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + columns.map(col => { + col._2 match { + case value: List[Any] => + selectWhere.and(QueryBuilder.in(col._1, value.asJava)) + case _ => + selectWhere.and(QueryBuilder.eq(col._1, col._2)) + } + }) + logger.info("select query {}", selectWhere.toString) + cassandraUtil.find(selectWhere.toString).asScala.toList + } + } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index bffb7bb22..7e04d76bb 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -43,6 +43,11 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val dbBatchId = "batchid" val dbCourseId = "courseid" val dbUserId = "userid" + val contentHierarchyTable: String = "content_hierarchy" + val contentHierarchyKeySpace: String = "dev_hierarchy_store" + val Hierarchy: String = "hierarchy" + val childrens: String = "children" + val batches: String = "batches" //API URL val contentBasePath = config.getString("service.content.basePath") @@ -81,5 +86,6 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val userAccBlockedErrCode = "UOS_USRRED0006" val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false val programCertProgram: String = "program_cert_program" + val parentCollections: String = "parentCollections" } From ca23068fba110e120c5f15e46a10c54e9d9d66de Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 21 Sep 2023 01:52:53 +0530 Subject: [PATCH 03/88] Dev #104439 - Create new Flink job to validate the program certificate event generation -Kafka event changes --- .../functions/ProgramCertPreProcessorFn.scala | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 7c9ce8a1b..663984578 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -11,7 +11,7 @@ import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.exception.InvalidEventException import org.sunbird.job.programcert.domain.Event import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} import scala.collection.JavaConverters._ @@ -78,7 +78,32 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } } if(isProgramCertificateToBeGenerated) { - //Need to add kafka event to generate Certificate + //Add kafka event to generate Certificate for Program + val eData = Map[String, AnyRef]( + "eid" -> "BE_JOB_REQUEST", + "ets" -> "1695019553226", + "mid" -> "LP.1695019553226.dcfd4458-f7e4-4e23-bfb7-9531ec91a1fe", + "actor" -> Map( + "id" -> "Program Certificate Generator", + "type" -> "System"), + "context" -> Map( + "pdata" -> Map( + "ver" -> "1.0", + "id" -> "org.sunbird.platform")), + "object" -> Map( + "id" -> event.batchId.concat("_").concat(parentCourse), + "type" -> "ProgramCertificateGeneration"), + "edata" -> Map( + "userIds" -> List(event.userId), + "action" -> "issue-certificate", + "iteration" -> 1, + "trigger" -> "auto-issue", + "batchId" -> event.batchId, + "reIssue" -> false, + "courseId" -> parentCourse)) + + val requestBody = JSONUtil.serialize(eData) + context.output(config.generateCertificateOutputTag, requestBody) } } } From d76f191056ac28a535da479b1cc5e207da9498b7 Mon Sep 17 00:00:00 2001 From: ravisaurav-tarento <142487505+ravisaurav-tarento@users.noreply.github.com> Date: Thu, 21 Sep 2023 09:42:47 +0530 Subject: [PATCH 04/88] 104436:Changes related to publishing event to Kafka after ceritificate generation (#22) --- .../org/sunbird/job/certgen/domain/Models.scala | 11 +++++++++++ .../functions/CertificateGeneratorFunction.scala | 16 ++++++++++++++++ .../task/CertificateGeneratorConfig.scala | 2 ++ 3 files changed, 29 insertions(+) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala index bb6baf196..5683709e2 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala @@ -64,3 +64,14 @@ case class UserEnrollmentData(batchId: String, case class Recipient(id: String, name: String, `type`: String) case class Training(id: String, name: String, `type`: String, batchId: String) case class Issuer(url: String, name: String, kid: String) +case class ActorObject(id: String = "Certificate Generator", `type`: String = "System") +case class EventObjectCourseCertificate(id: String, `type`: String = "GenerateCertificate") +case class EventContextCorseCertificate(pdata: Map[String, String] = Map("ver" -> "1.0", "id" -> "org.sunbird.learning.platform")) +case class BEJobRequestEvent(actor: ActorObject= ActorObject(), + eid: String = "BE_JOB_REQUEST", + edata: Map[String, AnyRef], + ets: Long = System.currentTimeMillis(), + context: EventContextCorseCertificate = EventContextCorseCertificate(), + mid: String = s"LMS.${UUID.randomUUID().toString}", + `object`: EventObjectCourseCertificate + ) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index a8d3a92d6..98ac61c42 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -29,6 +29,7 @@ import java.util import java.util.stream.Collectors import java.util.{Base64, Date} import scala.collection.JavaConverters._ +import org.sunbird.job.certgen.domain.{ BEJobRequestEvent, EventObjectCourseCertificate} class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, storageService: StorageService, @transient var cassandraUtil: CassandraUtil = null) extends BaseProcessKeyedFunction[String, Event, String](config) { @@ -77,6 +78,11 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: metrics.incCounter(config.skippedEventCount) logger.info(s"Certificate already issued for: ${event.eData.getOrElse("userId", "")} ${event.related}") } + val courseCompletionEvent = generateCourseCompletionEvent(event) + Option(courseCompletionEvent ).map(e => { + context.output(config.generateCertificateOutputTag, courseCompletionEvent) + metrics.incCounter(config.successEventCount) + }) } catch { case e: Exception => metrics.incCounter(config.failedEventCount) @@ -358,4 +364,14 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: } + def generateCourseCompletionEvent(event: Event) = { + val eData = Map[String, AnyRef]( + "userId" -> event.userId, + "batchId" -> event.batchId, + "courseId" -> event.courseId + ) + ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObjectCourseCertificate(id = event.userId))) + } + + } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index e5793226d..dc341fcb8 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -164,5 +164,7 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val userFeedMsg: String = "You have earned a certificate! Download it from your profile page." val priorityValue = 1 val userFeedCount = "user-feed-count" + val generateCertificateOutputTagName = "generate-certificate-request" + val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) } From 43a629b33d33998529f9c6b92f1bf6372313a917 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 21 Sep 2023 10:06:42 +0530 Subject: [PATCH 05/88] Dev #104439 - Create new Flink job to validate the program certificate event generation -pom.xml changes --- .../program-cert-pre-processor/pom.xml | 215 ++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 credential-generator/program-cert-pre-processor/pom.xml diff --git a/credential-generator/program-cert-pre-processor/pom.xml b/credential-generator/program-cert-pre-processor/pom.xml new file mode 100644 index 000000000..380842673 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/pom.xml @@ -0,0 +1,215 @@ + + + 4.0.0 + + org.sunbird + credential-generator + 1.0 + + org.sunbird + program-cert-pre-processor + 1.0-SNAPSHOT + jar + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.version} + ${flink.version} + provided + + + org.sunbird + jobs-core + 1.0.0 + + + joda-time + joda-time + 2.10.6 + + + org.sunbird + jobs-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_${scala.version} + ${flink.version} + test + + + org.apache.flink + flink-runtime_${scala.version} + ${flink.version} + test + tests + + + org.apache.flink + flink-streaming-java_${scala.version} + ${flink.version} + test + tests + + + org.scalatest + scalatest_${scala.version} + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + it.ozimov + embedded-redis + 0.7.1 + test + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.job.programcert.task.ProgramCertPreProcessorTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + 11 + 11 + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + collection-complete-post-processor-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + \ No newline at end of file From 75ca406fecb7bc15596af4cb56c8bc979f440ab1 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 22 Sep 2023 00:24:26 +0530 Subject: [PATCH 06/88] Dev #104439 - Create new Flink job to validate the program certificate event generation -Added the code changes for Adding logic --- .../functions/IssueCertificateHelper.scala | 6 ++-- .../functions/ProgramCertPreProcessorFn.scala | 36 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala index 1c3eb4bd9..d23482e8b 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala @@ -31,14 +31,14 @@ trait IssueCertificateHelper { } } - def getCourseReadDetails(courseId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): List[String] = { + def getCourseReadDetails(courseId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.ArrayList[String] = { val courseMetadata = cache.getWithRetry(courseId) if (null == courseMetadata || courseMetadata.isEmpty) { val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=primaryCategory,identifier,parentCollections" val response = getAPICall(url, "content")(config, httpUtil, metrics) - response.getOrElse(config.parentCollections, List()).asInstanceOf[List[String]] + response.getOrElse(config.parentCollections, new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] } else { - courseMetadata.getOrElse(config.parentCollections, List()).asInstanceOf[List[String]] + courseMetadata.getOrElse(config.parentCollections.toLowerCase(), new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] } } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 663984578..4918670e2 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -3,8 +3,11 @@ package org.sunbird.job.programcert.functions import com.datastax.driver.core.querybuilder.QueryBuilder import com.datastax.driver.core.{Row, TypeTokens} import com.google.common.reflect.TypeToken +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.slf4j.LoggerFactory import org.sunbird.job.cache.{DataCache, RedisConnect} @@ -26,6 +29,7 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) private var cache: DataCache = _ private var contentCache: DataCache = _ + lazy private val mapper: ObjectMapper = new ObjectMapper() override def open(parameters: Configuration): Unit = { super.open(parameters) @@ -55,23 +59,23 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: context: KeyedProcessFunction[String, Event, String]#Context, metrics: Metrics): Unit = { try { - val getParentForCourse = getCourseReadDetails(event.courseId)(metrics, config, contentCache, httpUtil).asInstanceOf[List[String]] + val getParentForCourse = getCourseReadDetails(event.courseId)(metrics, config, contentCache, httpUtil).asInstanceOf[java.util.List[String]] if(!getParentForCourse.isEmpty) { for (parentCourse <- getParentForCourse) { - val childrenForCuratedProgram = getProgramChildren(parentCourse)(metrics, config, contentCache, httpUtil).asInstanceOf[Map[String, AnyRef]] + val childrenForCuratedProgram = getProgramChildren(parentCourse)(metrics, config, contentCache, httpUtil) if (!childrenForCuratedProgram.isEmpty) { - val contentDataForProgram = childrenForCuratedProgram.get(config.childrens).asInstanceOf[List[Map[String, AnyRef]]] + val contentDataForProgram = childrenForCuratedProgram.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] var isProgramCertificateToBeGenerated: Boolean = true; for (childNode <- contentDataForProgram) { val courseId: String = childNode.get(config.identifier).asInstanceOf[String] - val batchesForCourse: java.util.List[java.util.Map[String, AnyRef]] = childNode.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + /* val batchesForCourse: java.util.List[java.util.Map[String, AnyRef]] = childNode.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] val filteredBatches = batchesForCourse.filter(batch => batch.get("status") != 2).toList - val batchId: String = filteredBatches.get(0).get("batchId").asInstanceOf[String] + val batchId: String = filteredBatches.get(0).get("batchId").asInstanceOf[String]*/ val userId: String = event.userId - val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbBatchId.toLowerCase -> batchId, config.dbCourseId.toLowerCase -> courseId) + val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) val isCertificateIssued: List[Row] = getCourseIssuedCertificateForUser(primaryFields)(metrics).asInstanceOf[List[Row]] breakable { - if (isCertificateIssued == null) { + if (isCertificateIssued == null || isCertificateIssued.isEmpty) { isProgramCertificateToBeGenerated = false; break } @@ -116,16 +120,17 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: logger.info("Inside the Process ElementForProgram"); } - def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): Map[String, AnyRef] = { - val query = QueryBuilder.select().all().from(config.contentHierarchyKeySpace, config.contentHierarchyTable) + def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { + val query = QueryBuilder.select(config.Hierarchy).from(config.contentHierarchyKeySpace, config.contentHierarchyTable) .where(QueryBuilder.eq(config.identifier, programId)) - val row: Row = cassandraUtil.findOne(query.toString) - if (null != row) { - val templates = row.getMap(config.Hierarchy, TypeToken.of(classOf[String]), TypeTokens.mapOf(classOf[String], classOf[String])) - templates.asScala.map(template => (template._1 -> template._2.asScala.toMap)).toMap - } else { - Map[String, Map[String, String]]() + val row = cassandraUtil.find(query.toString) + if (CollectionUtils.isNotEmpty(row)) { + val hierarchy = row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] + if (StringUtils.isNotBlank(hierarchy)) + mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) + else new java.util.HashMap[String, AnyRef]() } + else new java.util.HashMap[String, AnyRef]() } private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics) = { @@ -144,5 +149,4 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: logger.info("select query {}", selectWhere.toString) cassandraUtil.find(selectWhere.toString).asScala.toList } - } From ec2301b815ff8f512d34739f55be67da2b7eee20 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 22 Sep 2023 11:21:03 +0530 Subject: [PATCH 07/88] Updated changes --- .../functions/IssueCertificateHelper.scala | 35 +++++++++++++++++-- .../CollectionCertPreProcessorConfig.scala | 3 +- .../sunbird/job/certgen/domain/Event.scala | 4 +++ .../CertificateGeneratorFunction.scala | 28 ++++++++++----- .../task/CertificateGeneratorConfig.scala | 9 +++-- .../task/CertificateGeneratorStreamTask.scala | 5 +++ .../job/programcert/domain/Event.scala | 2 +- .../task/ProgramCertPreProcessorConfig.scala | 2 +- 8 files changed, 72 insertions(+), 16 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 3f946a1b5..debd031de 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -210,7 +210,8 @@ trait IssueCertificateHelper { val lastName = Option(userDetails.getOrElse("lastName", "").asInstanceOf[String]).getOrElse("") def nullStringCheck(name:String):String = {if(StringUtils.equalsIgnoreCase("null", name)) "" else name} val recipientName = nullStringCheck(firstName).concat(" ").concat(nullStringCheck(lastName)).trim - val courseName = getCourseName(event.courseId)(metrics, config, cache, httpUtil) + val courseInfo: Map[String, Any] = getCourseInfo(event.courseId)(metrics, config, cache, httpUtil) + val courseName = courseInfo.getOrElse("courseName", "").asInstanceOf[String] val dateFormatter = new SimpleDateFormat("yyyy-MM-dd") val related = getRelatedData(event, enrolledUser, assessedUser, userDetails, additionalProps, certName, courseName)(config) val providerName = getCourseOrganisation(event.courseId)(metrics, config, cache, httpUtil) @@ -230,7 +231,9 @@ trait IssueCertificateHelper { "related" -> related, "name" -> certName, "providerName" -> providerName, - "tag" -> event.batchId + "tag" -> event.batchId, + "primaryCategory" -> courseInfo.getOrElse("primaryCategory", "").asInstanceOf[String], + "parentCollections" -> courseInfo.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] ) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) @@ -252,4 +255,32 @@ trait IssueCertificateHelper { Map[String, Any]("batchId" -> event.batchId, "courseId" -> event.courseId, "type" -> certName) ++ locationProps ++ enrolledUser.additionalProps ++ assessedUser.additionalProps ++ userAdditionalProps ++ courseAdditionalProps } + + def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): Map[String, Any] = { + val courseMetadata = cache.getWithRetry(courseId) + val courseInfoMap = if (null == courseMetadata || courseMetadata.isEmpty) { + val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + Map( + "courseId" -> courseId, + "courseName" -> courseName, + "parentCollections" -> parentCollections, + "primaryCategory" -> primaryCategory + ) + } else { + val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = courseMetadata.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + Map( + "courseId" -> courseId, + "courseName" -> courseName, + "parentCollections" -> parentCollections, + "primaryCategory" -> primaryCategory + ) + } + courseInfoMap + } } diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala index f986cfc84..36eccf4a7 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala @@ -79,6 +79,7 @@ class CollectionCertPreProcessorConfig(override val config: Config) extends Base val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") val userAccBlockedErrCode = "UOS_USRRED0006" val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false - + val primaryCategory: String = "primaryCategory" + val parentCollections: String = "parentCollections" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala index cb08ab00c..7f7531e3a 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala @@ -60,4 +60,8 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def providerName: String = readOrDefault[String]("edata.providerName", "") + def primaryCategory: String = readOrDefault[String]("edata.primaryCategory", "") + + def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index 98ac61c42..679e83c8e 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -27,7 +27,7 @@ import java.lang.reflect.Type import java.text.SimpleDateFormat import java.util import java.util.stream.Collectors -import java.util.{Base64, Date} +import java.util.{Base64, Date, UUID} import scala.collection.JavaConverters._ import org.sunbird.job.certgen.domain.{ BEJobRequestEvent, EventObjectCourseCertificate} @@ -78,11 +78,14 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: metrics.incCounter(config.skippedEventCount) logger.info(s"Certificate already issued for: ${event.eData.getOrElse("userId", "")} ${event.related}") } - val courseCompletionEvent = generateCourseCompletionEvent(event) - Option(courseCompletionEvent ).map(e => { - context.output(config.generateCertificateOutputTag, courseCompletionEvent) - metrics.incCounter(config.successEventCount) - }) + logger.info("Certificate Issued for '"+ event.primaryCategory + "', parentCollections value ? " + event.parentCollections) + if ("Course".equalsIgnoreCase(event.primaryCategory) && !event.parentCollections.isEmpty) { + val courseCompletionEvent = generateCourseCompletionEvent(event) + Option(courseCompletionEvent ).map(e => { + context.output(config.generateProgramCertificateOutputTag, courseCompletionEvent) + }) + } + metrics.incCounter(config.successEventCount) } catch { case e: Exception => metrics.incCounter(config.failedEventCount) @@ -368,10 +371,19 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val eData = Map[String, AnyRef]( "userId" -> event.userId, "batchId" -> event.batchId, - "courseId" -> event.courseId + "courseId" -> event.courseId, + "parentCollections" -> event.parentCollections ) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObjectCourseCertificate(id = event.userId))) } - + /* + def createProgramCertPreProcessorEvent(event: Event, context: KeyedProcessFunction[String, Event, String]#Context) : Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Program Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${event.batchId}_${event.courseId}","type": "ProgramCertificateGeneration"},"edata": {"userIds": ["${event.userId}"],"action": "program_cert_pre_process","iteration": 1, "trigger": "auto-issue","batchId": "${event.batchId}","reIssue": false,"courseId": "${event.courseId}"}}""" + logger.info("Cert generator... Triggering program cert pre processor event : " + event) + context.output(config.generateProgramCertificateOutputTag, event) + } + */ } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index dc341fcb8..2328eeb9d 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -21,17 +21,18 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon // Kafka Topics Configuration val kafkaInputTopic: String = config.getString("kafka.input.topic") val kafkaAuditEventTopic: String = config.getString("kafka.output.audit.topic") + val kafkaProgramCertOutputTopic: String = config.getString("kafka.program.cert.output.topic") val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false val enableRcCertificate: Boolean = if(config.hasPath("enable.rc.certificate")) config.getBoolean("enable.rc.certificate") else false - // Producers val certificateGeneratorAuditProducer = "collection-certificate-generator-audit-events-sink" override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") val notifierParallelism: Int = if(config.hasPath("task.notifier.parallelism")) config.getInt("task.notifier.parallelism") else 1 val userFeedParallelism: Int = if(config.hasPath("task.userfeed.parallelism")) config.getInt("task.userfeed.parallelism") else 1 + val generateProgramCertificateParallelism: Int = if(config.hasPath("task.programcert.parallelism")) config.getInt("task.programcert.parallelism") else 1 //ES configuration val esConnection: String = config.getString("es.basePath") @@ -164,7 +165,9 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val userFeedMsg: String = "You have earned a certificate! Download it from your profile page." val priorityValue = 1 val userFeedCount = "user-feed-count" - val generateCertificateOutputTagName = "generate-certificate-request" - val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) + val generateProgramCertificateOutputTagName: String = "generate-program-certificate-request" + val generateProgramCertificateOutputTag: OutputTag[String] = OutputTag[String](generateProgramCertificateOutputTagName) + + val generateProgramCertificateProducer = "generate-program-certificate-sink" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala index 5b7784e1f..00a409bcb 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala @@ -55,6 +55,11 @@ class CertificateGeneratorStreamTask(config: CertificateGeneratorConfig, kafkaCo .uid("user-feed") .setParallelism(config.userFeedParallelism) + processStreamTask.getSideOutput(config.generateProgramCertificateOutputTag) + .addSink(kafkaConnector.kafkaStringSink(config.kafkaProgramCertOutputTopic)) + .name(config.generateProgramCertificateProducer) + .uid(config.generateProgramCertificateProducer) + .setParallelism(config.generateProgramCertificateParallelism) env.execute(config.jobName) } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala index 2550d77a1..7ffcc0992 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala @@ -19,7 +19,7 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def eData: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("edata", Map[String, AnyRef]()) def isValid()(config: ProgramCertPreProcessorConfig): Boolean = { - config.programCertProgram.equalsIgnoreCase(action) && !batchId.isEmpty && !courseId.isEmpty && + config.programCertPreProcess.equalsIgnoreCase(action) && !batchId.isEmpty && !courseId.isEmpty && !userId.isEmpty } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index 7e04d76bb..36a81e064 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -85,7 +85,7 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") val userAccBlockedErrCode = "UOS_USRRED0006" val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false - val programCertProgram: String = "program_cert_program" + val programCertPreProcess: String = "program_cert_pre_process" val parentCollections: String = "parentCollections" } From 19cb9332fb536a786dff318b60442d48e22bdd02 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 22 Sep 2023 13:58:40 +0530 Subject: [PATCH 08/88] Dev #104439 - Create new Flink job to validate the program certificate event generation -Added the changes for batches Validation and Enrolment Validation --- .../resources/program-cert-pre-processor.conf | 8 +- .../job/programcert/domain/Event.scala | 12 +- .../functions/ProgramCertPreProcessorFn.scala | 112 ++++++++++-------- .../task/ProgramCertPreProcessorConfig.scala | 24 +--- 4 files changed, 76 insertions(+), 80 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf index 50cff9b9d..9f8116582 100644 --- a/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf +++ b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf @@ -1,10 +1,10 @@ include "base-config.conf" kafka { - input.topic = "sunbirddev.program.cert.request" - output.topic = "sunbirddev.program.cert.output.request" - output.failed.topic = "sunbirddev.issue.certificate.failed" - groupId = "collection-cert-pre-processor-group" + input.topic = "dev.issue.program.certificate.request" + output.topic = "dev.issue.certificate.request" + output.failed.topic = "dev.issue.certificate.failed" + groupId = "dev-program-cert-pre-processor-group" } task { diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala index 7ffcc0992..9f3a46640 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala @@ -11,10 +11,14 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def courseId: String = readOrDefault[String]("edata.courseId", "") - def userId: String = { - val list = readOrDefault[List[String]]("edata.userIds", List[String]()) - if(list.isEmpty) "" else list.head - } + def userId: String = readOrDefault[String]("edata.userId", "") + + def providerName: String = readOrDefault[String]("edata.providerName", "") + + def primaryCategory: String = readOrDefault[String]("edata.primaryCategory", "") + + def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + def eData: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("edata", Map[String, AnyRef]()) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 4918670e2..64eff0638 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -1,8 +1,7 @@ package org.sunbird.job.programcert.functions -import com.datastax.driver.core.querybuilder.QueryBuilder -import com.datastax.driver.core.{Row, TypeTokens} -import com.google.common.reflect.TypeToken +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.typeinfo.TypeInformation @@ -41,7 +40,6 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) contentCache.init() - logger.info("This is the flink testing") } override def close(): Unit = { @@ -59,55 +57,58 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: context: KeyedProcessFunction[String, Event, String]#Context, metrics: Metrics): Unit = { try { - val getParentForCourse = getCourseReadDetails(event.courseId)(metrics, config, contentCache, httpUtil).asInstanceOf[java.util.List[String]] - if(!getParentForCourse.isEmpty) { - for (parentCourse <- getParentForCourse) { - val childrenForCuratedProgram = getProgramChildren(parentCourse)(metrics, config, contentCache, httpUtil) - if (!childrenForCuratedProgram.isEmpty) { - val contentDataForProgram = childrenForCuratedProgram.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] - var isProgramCertificateToBeGenerated: Boolean = true; - for (childNode <- contentDataForProgram) { - val courseId: String = childNode.get(config.identifier).asInstanceOf[String] - /* val batchesForCourse: java.util.List[java.util.Map[String, AnyRef]] = childNode.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] - val filteredBatches = batchesForCourse.filter(batch => batch.get("status") != 2).toList - val batchId: String = filteredBatches.get(0).get("batchId").asInstanceOf[String]*/ - val userId: String = event.userId - val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) - val isCertificateIssued: List[Row] = getCourseIssuedCertificateForUser(primaryFields)(metrics).asInstanceOf[List[Row]] - breakable { - if (isCertificateIssued == null || isCertificateIssued.isEmpty) { - isProgramCertificateToBeGenerated = false; - break + val getParentIdForCourse = event.parentCollections + if (!getParentIdForCourse.isEmpty) { + for (courseParentId <- getParentIdForCourse) { + val curatedProgramHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) + if (!curatedProgramHierarchy.isEmpty) { + val batchesForProgram: java.util.List[java.util.Map[String, AnyRef]] = curatedProgramHierarchy.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + val filteredBatches = batchesForProgram.filter(batch => batch.get("status") != 2).toList + val batchId: String = filteredBatches.get(0).get(config.dbBatchId).asInstanceOf[String] + val programEnrollmentStatus = getEnrolment(event.userId, courseParentId, batchId)(metrics) + if (null != programEnrollmentStatus) { + val contentDataForProgram = curatedProgramHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] + var isProgramCertificateToBeGenerated: Boolean = true; + for (childNode <- contentDataForProgram) { + val courseId: String = childNode.get(config.identifier).asInstanceOf[String] + val userId: String = event.userId + val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) + val isCertificateIssued: List[Row] = getCourseIssuedCertificateForUser(primaryFields)(metrics) + breakable { + if (isCertificateIssued == null || isCertificateIssued.isEmpty) { + isProgramCertificateToBeGenerated = false; + break + } } } - } - if(isProgramCertificateToBeGenerated) { - //Add kafka event to generate Certificate for Program - val eData = Map[String, AnyRef]( - "eid" -> "BE_JOB_REQUEST", - "ets" -> "1695019553226", - "mid" -> "LP.1695019553226.dcfd4458-f7e4-4e23-bfb7-9531ec91a1fe", - "actor" -> Map( - "id" -> "Program Certificate Generator", - "type" -> "System"), - "context" -> Map( - "pdata" -> Map( - "ver" -> "1.0", - "id" -> "org.sunbird.platform")), - "object" -> Map( - "id" -> event.batchId.concat("_").concat(parentCourse), - "type" -> "ProgramCertificateGeneration"), - "edata" -> Map( - "userIds" -> List(event.userId), - "action" -> "issue-certificate", - "iteration" -> 1, - "trigger" -> "auto-issue", - "batchId" -> event.batchId, - "reIssue" -> false, - "courseId" -> parentCourse)) + if (isProgramCertificateToBeGenerated) { + //Add kafka event to generate Certificate for Program + val eData = Map[String, AnyRef]( + "eid" -> "BE_JOB_REQUEST", + "ets" -> "1695019553226", + "mid" -> "LP.1695019553226.dcfd4458-f7e4-4e23-bfb7-9531ec91a1fe", + "actor" -> Map( + "id" -> "Program Certificate Generator", + "type" -> "System"), + "context" -> Map( + "pdata" -> Map( + "ver" -> "1.0", + "id" -> "org.sunbird.platform")), + "object" -> Map( + "id" -> batchId.concat("_").concat(courseParentId), + "type" -> "ProgramCertificateGeneration"), + "edata" -> Map( + "userIds" -> List(event.userId), + "action" -> "issue-certificate", + "iteration" -> 1, + "trigger" -> "auto-issue", + "batchId" -> batchId, + "reIssue" -> false, + "courseId" -> courseParentId)) - val requestBody = JSONUtil.serialize(eData) - context.output(config.generateCertificateOutputTag, requestBody) + val requestBody = JSONUtil.serialize(eData) + context.output(config.generateCertificateOutputTag, requestBody) + } } } } @@ -149,4 +150,15 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: logger.info("select query {}", selectWhere.toString) cassandraUtil.find(selectWhere.toString).asScala.toList } + + def getEnrolment(userId: String, programId: String, batchId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", programId)) + .and(QueryBuilder.eq("batchid", batchId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index 36a81e064..dc7d5f467 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -32,15 +32,10 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob //Cassandra config val dbHost: String = config.getString("lms-cassandra.host") - /*val dbHost: String = "192.168.56.101" - val dbPort: Int = 9042*/ val dbPort: Int = config.getInt("lms-cassandra.port") val keyspace: String = config.getString("lms-cassandra.keyspace") - val courseTable: String = config.getString("lms-cassandra.course_batch.table") val userEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") - val assessmentTable: String = config.getString("lms-cassandra.assessment_aggregator.table") - val useActivityAggTable: String = config.getString("lms-cassandra.user_activity_agg.table") - val dbBatchId = "batchid" + val dbBatchId = "batchId" val dbCourseId = "courseid" val dbUserId = "userid" val contentHierarchyTable: String = "content_hierarchy" @@ -53,7 +48,7 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val contentBasePath = config.getString("service.content.basePath") val learnerBasePath = config.getString("service.learner.basePath") val userReadApi = config.getString("user_read_api") - val contentReadApi = config.getString("content_read_api") + val contentReadApi = "/content/v4/read" // Metric List val totalEventsCount = "total-events-count" @@ -65,26 +60,11 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val cacheHitCount = "cache-hit-cout" //Constants - val issueCertificate = "issue-certificate" - val certTemplates = "cert_templates" - val criteria: String = "criteria" - val enrollment: String = "enrollment" - val assessment: String = "assessment" - val users: String = "users" - val active: String = "active" - val issuedCertificates: String = "issued_certificates" val status: String = "status" val name: String = "name" - val user: String= "user" val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") val identifier: String = "identifier" - val completedOn: String = "completedon" - val issuer: String = "issuer" - val signatoryList: String = "signatoryList" - val certBasePath: String = config.getString("cert_domain_url") + "/certs" - val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") val userAccBlockedErrCode = "UOS_USRRED0006" - val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false val programCertPreProcess: String = "program_cert_pre_process" val parentCollections: String = "parentCollections" From cf735fa2f30d51e5f3c86cb619711bcae3fc44d1 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 22 Sep 2023 16:55:36 +0530 Subject: [PATCH 09/88] Dev #104439 - Create new Flink job to validate the program certificate event generation Made the changes for the batches Read --- .../functions/ProgramCertPreProcessorFn.scala | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 64eff0638..6e326cfe2 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -60,13 +60,12 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: val getParentIdForCourse = event.parentCollections if (!getParentIdForCourse.isEmpty) { for (courseParentId <- getParentIdForCourse) { - val curatedProgramHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) - if (!curatedProgramHierarchy.isEmpty) { - val batchesForProgram: java.util.List[java.util.Map[String, AnyRef]] = curatedProgramHierarchy.get(config.batches).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] - val filteredBatches = batchesForProgram.filter(batch => batch.get("status") != 2).toList - val batchId: String = filteredBatches.get(0).get(config.dbBatchId).asInstanceOf[String] - val programEnrollmentStatus = getEnrolment(event.userId, courseParentId, batchId)(metrics) - if (null != programEnrollmentStatus) { + val programEnrollmentStatus = getEnrolment(event.userId, courseParentId)(metrics) + //if enrolled into program + if (null != programEnrollmentStatus) { + val curatedProgramHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) + if (!curatedProgramHierarchy.isEmpty) { + val batchId: String = programEnrollmentStatus.getString(config.dbBatchId) val contentDataForProgram = curatedProgramHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] var isProgramCertificateToBeGenerated: Boolean = true; for (childNode <- contentDataForProgram) { @@ -151,14 +150,23 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: cassandraUtil.find(selectWhere.toString).asScala.toList } - def getEnrolment(userId: String, programId: String, batchId: String)(implicit metrics: Metrics) = { + def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { val selectWhere: Select.Where = QueryBuilder.select().all() .from(config.keyspace, config.userEnrolmentsTable). where() - selectWhere.and(QueryBuilder.eq("userid", userId)) - .and(QueryBuilder.eq("courseid", programId)) - .and(QueryBuilder.eq("batchid", batchId)) + selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) + .and(QueryBuilder.eq(config.dbCourseId, programId)) metrics.incCounter(config.dbReadCount) - cassandraUtil.findOne(selectWhere.toString) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + row = row.filter(x => x.getInt("status") != 2).toList + if (row.size() == 1) { + row.asScala.get(0) + } else { + null + } + } else { + null + } } } From 8e7fc618a09dbb3939134eaf3974bc7aefb7b58c Mon Sep 17 00:00:00 2001 From: Karthikeyan Rajendran <70887864+karthik-tarento@users.noreply.github.com> Date: Mon, 25 Sep 2023 09:46:46 +0530 Subject: [PATCH 10/88] 4.8.6.1 post publish flink (#27) * Initial Commit 1. postPublicRelationUpdateStream Added Kafka consumer and producer. 2. Added the configuration in the PostPublishProcessorConfig. 3. Added service method postPublishRelationUpdate to the BatchCreation. 4. Added a function classs PostPublishRelationUpdaterFunction. 5. Added the configuration in the Post-publish-processor.conf * Code changes Code changes * Dev #104434-Enhance post publish processor flink job and add parentCollections details against Course. Code Review fixes. * Added method to get course from redis * Dev #104434- Enhance post publish processor flink job and add parentCollections details against Course. Code Fixes * Dev #104434-Enhance post publish processor flink job and add parentCollections details against Course Content State Url Updated. * Updated code. Using proper system update API * Optimized code * Formatted code * Formatted code * reading keyspace from config * reading keyspace from config --------- Co-authored-by: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> --- .../resources/post-publish-processor.conf | 3 + .../functions/PostPublishEventRouter.scala | 9 +- .../PostPublishRelationUpdaterFunction.scala | 191 ++++++++++++++++++ .../postpublish/helpers/BatchCreation.scala | 1 - .../helpers/PostPublishRelationUpdater.scala | 130 ++++++++++++ .../task/PostPublishProcessorConfig.scala | 30 ++- .../task/PostPublishProcessorStreamTask.scala | 3 + 7 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala create mode 100644 post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala diff --git a/post-publish-processor/src/main/resources/post-publish-processor.conf b/post-publish-processor/src/main/resources/post-publish-processor.conf index d214660e0..2d7391099 100644 --- a/post-publish-processor/src/main/resources/post-publish-processor.conf +++ b/post-publish-processor/src/main/resources/post-publish-processor.conf @@ -17,11 +17,13 @@ task { shallow_copy.parallelism = 1 link_dialcode.parallelism = 1 batch_create.parallelism = 1 + post-publish-relation-update.parallelism = 1 } lms-cassandra { keyspace = "sunbird_courses" batchTable = "course_batch" + hierarchyStoreKeySpace = "dev_hierarchy_store" } dialcode-cassandra { @@ -34,6 +36,7 @@ service { lms.basePath = "http://11.2.6.6/lms" learning_service.basePath = "http://11.2.4.22:8080/learning-service" dial.basePath = "https://dev.sunbirded.org/dial/" + content.basePath = "http://11.2.6.6/content" } dialcode { diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala index f37a06c63..2c6cb4d50 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala @@ -4,8 +4,9 @@ import com.google.gson.reflect.TypeToken import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.helpers.{BatchCreation, DialHelper, ShallowCopyPublishing} +import org.sunbird.job.postpublish.helpers.{BatchCreation, DialHelper, PostPublishRelationUpdater, ShallowCopyPublishing} import org.sunbird.job.postpublish.task.PostPublishProcessorConfig import org.sunbird.job.util.{CassandraUtil, HttpUtil, Neo4JUtil} import org.sunbird.job.{BaseProcessFunction, Metrics} @@ -17,7 +18,7 @@ case class PublishMetadata(identifier: String, contentType: String, mimeType: St class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpUtil, @transient var neo4JUtil: Neo4JUtil = null, @transient var cassandraUtil: CassandraUtil = null) - extends BaseProcessFunction[Event, String](config) with ShallowCopyPublishing with BatchCreation with DialHelper { + extends BaseProcessFunction[Event, String](config) with ShallowCopyPublishing with BatchCreation with DialHelper with PostPublishRelationUpdater { private[this] val logger = LoggerFactory.getLogger(classOf[PostPublishEventRouter]) val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType @@ -52,6 +53,10 @@ class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpU val dialCodeDetails = getDialCodeDetails(identifier, event)(neo4JUtil, config) if (!dialCodeDetails.isEmpty) context.output(config.linkDIALCodeOutTag, dialCodeDetails) + + //Process Post Publish Relation Update + context.output(config.postPublishRelationUpdateOutTag, null) + } else { metrics.incCounter(config.skippedEventCount) logger.info(s"Event not qualified for publishing for Identifier : ${event.collectionId}.") diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala new file mode 100644 index 000000000..ec25e0a00 --- /dev/null +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -0,0 +1,191 @@ +package org.sunbird.job.postpublish.functions + +import com.datastax.driver.core.querybuilder.QueryBuilder +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.configuration.Configuration +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.http.client.methods.HttpPatch +import org.apache.http.entity.{ContentType, StringEntity} +import org.apache.http.impl.client.HttpClients +import org.apache.http.{HttpResponse, StatusLine} +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.APIException +import org.sunbird.job.postpublish.helpers.PostPublishRelationUpdater +import org.sunbird.job.postpublish.task.PostPublishProcessorConfig +import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} +import org.sunbird.job.{BaseProcessFunction, Metrics} + +import java.time.format.DateTimeFormatter +import java.time.{ZoneId, ZonedDateTime} +import java.util +import scala.collection.JavaConverters._ +import scala.collection.mutable.ListBuffer + +/** @author + * mahesh.vakkund + */ +class PostPublishRelationUpdaterFunction( + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + @transient var cassandraUtil: CassandraUtil = null +) extends BaseProcessFunction[java.util.Map[String, AnyRef], String](config) + with PostPublishRelationUpdater { + + private[this] val logger = + LoggerFactory.getLogger(classOf[PostPublishRelationUpdaterFunction]) + lazy private val mapper: ObjectMapper = new ObjectMapper() + private var cache: DataCache = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + val redisConnect = new RedisConnect(config) + cache = + new DataCache(config, redisConnect, config.contentCacheStore, List()) + cache.init() + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + private def postPublishRelationUpdate( + eData: util.Map[String, AnyRef] + )(implicit + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + cassandraUtil: CassandraUtil, + metrics: Metrics + ): Unit = { + val programHierarchy = getProgramHierarchy( + eData.get("identifier").asInstanceOf[String] + )(metrics, config, cache, httpUtil) + if (programHierarchy.isEmpty) { + logger.info( + "PostPublishRelationUpdaterFunction :: Failed to get program Hierarchy." + ) + return + } + + val childCourseList = programHierarchy + .get(config.children) + .asInstanceOf[List[Map[String, AnyRef]]] + for (childNode <- childCourseList) { + val primaryCategory: String = + childNode.get(config.primaryCategory).asInstanceOf[String] + if (primaryCategory.equalsIgnoreCase("Course")) { + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo( + childNode.get("identifier").asInstanceOf[String] + )(metrics, config, cache, httpUtil) + val parentCollections: ListBuffer[String] = contentObj + .getOrDefault(config.parentCollections, ListBuffer.empty[String]) + .asInstanceOf[ListBuffer[String]] + val versionKey: String = + contentObj.get(config.versionKey).asInstanceOf[String] + val identifier: String = + childNode.get("identifier").asInstanceOf[String] + if (parentCollections.isEmpty) { + parentCollections += identifier + } else { + if (!parentCollections.contains(identifier)) { + parentCollections += identifier + } + } + val requestData: Map[String, Any] = Map( + "request" -> Map( + "content" -> Map( + "versionKey" -> versionKey, + "parentCollections" -> parentCollections.toList + ) + ) + ) + val jsonString: String = JSONUtil.serialize(requestData) + val patchRequest = new HttpPatch( + config.contentSystemUpdatePath + identifier + ) + patchRequest.setEntity( + new StringEntity(jsonString, ContentType.APPLICATION_JSON) + ) + val httpClient = HttpClients.createDefault() + val response: HttpResponse = httpClient.execute(patchRequest) + val statusLine: StatusLine = response.getStatusLine + val statusCode: Int = statusLine.getStatusCode + if (statusCode == 200) { + logger.info("Processed the request.") + } else { + logger.error( + "Received error response for system update API. Response: " + JSONUtil + .serialize(response) + ) + } + } + } + } + + def getProgramHierarchy(programId: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val query = QueryBuilder + .select(config.Hierarchy) + .from(config.hierarchyStoreKeySpace, config.contentHierarchyTable) + .where(QueryBuilder.eq(config.identifier, programId)) + val row = cassandraUtil.find(query.toString) + if (CollectionUtils.isNotEmpty(row)) { + val hierarchy = + row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] + if (StringUtils.isNotBlank(hierarchy)) + mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) + else new java.util.HashMap[String, AnyRef]() + } else new java.util.HashMap[String, AnyRef]() + } + + override def processElement( + eData: java.util.Map[String, AnyRef], + context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, + metrics: Metrics + ): Unit = { + val identifier = eData.getOrDefault("identifier", "").asInstanceOf[String] + val isValidProgram: Boolean = + verifyPrimaryCategory(identifier)(metrics, config, httpUtil, cache) + if (!isValidProgram) { + metrics.incCounter(config.postPublishRelationUpdateEventCount) + logger.info( + "PostPublishRelationUpdaterFunction:: started for Content : " + identifier + ) + try { + postPublishRelationUpdate(eData)( + config, + httpUtil, + cassandraUtil, + metrics + ) + metrics.incCounter(config.postPublishRelationUpdateSuccessCount) + logger.info( + "PostPublishRelationUpdaterFunction:: Completed for ContentId : " + identifier + ) + } catch { + case ex: Throwable => + logger.error( + s"Error while processing message for identifier : ${identifier}.", + ex + ) + metrics.incCounter(config.postPublishRelationUpdateFailureCount) + throw ex + } + } else { + logger.info( + "PostPublishRelationUpdaterFunction:: Nothing to do for ContentId : " + identifier + ) + } + } + + override def metricsList(): List[String] = ??? +} \ No newline at end of file diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala index 657e56556..3a66ccf67 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala @@ -9,7 +9,6 @@ import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, Neo4JUtil} import java.util import scala.collection.JavaConverters._ -import scala.collection.JavaConverters trait BatchCreation { diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala new file mode 100644 index 000000000..1abaa7fe1 --- /dev/null +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -0,0 +1,130 @@ +package org.sunbird.job.postpublish.helpers + +import org.apache.flink.configuration.Configuration +import org.slf4j.LoggerFactory +import org.sunbird.job.Metrics +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.APIException +import org.sunbird.job.postpublish.domain.Event +import org.sunbird.job.postpublish.task.PostPublishProcessorConfig +import org.sunbird.job.util._ + +/** @author + * mahesh.vakkund + */ +trait PostPublishRelationUpdater { + + private[this] val logger = + LoggerFactory.getLogger(classOf[PostPublishRelationUpdater]) + + def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (contentObj.isEmpty) { + contentObj.get("primaryCategory") match { + case Some("Program" | "Curated Program" | "Blended Program") => + isValidProgram = true + } + } + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + "/" + courseId + "?fields=identifier,name,versionKey,parentCollections,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + +} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala index 0b02f3620..cbca6e353 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala @@ -6,7 +6,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.scala.OutputTag import org.sunbird.job.BaseJobConfig import org.sunbird.job.postpublish.functions.PublishMetadata - +import scala.collection.immutable.Map import java.util class PostPublishProcessorConfig(override val config: Config) extends BaseJobConfig(config, "post-publish-processor") { @@ -30,6 +30,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val shallowCopyParallelism: Int = config.getInt("task.shallow_copy.parallelism") val linkDialCodeParallelism: Int = config.getInt("task.link_dialcode.parallelism") val batchCreateParallelism: Int = config.getInt("task.batch_create.parallelism") + val postPublishRelationUpdateParallelism: Int = config.getInt("task.post-publish-relation-update.parallelism") // Metric List val totalEventsCount = "total-events-count" @@ -42,6 +43,9 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val dialLinkSuccessCount = "dial-link-success-count" val dialLinkFailedCount = "dial-link-failed-count" val qrImageGeneratorEventCount = "qr-image-event-count" + val postPublishRelationUpdateEventCount = "post-publish-relation-update-count" + val postPublishRelationUpdateSuccessCount = "post-publish-relation-update-success-count" + val postPublishRelationUpdateFailureCount = "post-publish-relation-update-failure-count" // Cassandra Configurations val dbHost: String = config.getString("lms-cassandra.host") @@ -51,6 +55,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val defaultCertTemplateId = config.getString("lms-cassandra.certTemplateId") val sbSystemSettingsTableName = config.getString("lms-cassandra.systemSettingsTable") val batchTableName = config.getString("lms-cassandra.batchTable") + val hierarchyStoreKeySpace = config.getString("lms-cassandra.hierarchyStoreKeySpace") val dialcodeKeyspaceName = config.getString("dialcode-cassandra.keyspace") val dialcodeTableName = config.getString("dialcode-cassandra.imageTable") @@ -65,6 +70,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val shallowContentPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("shallow-copied-content-publish") val publishEventOutTag: OutputTag[String] = OutputTag[String]("content-publish-request") val generateQRImageOutTag: OutputTag[String] = OutputTag[String]("qr-image-generator-request") + val postPublishRelationUpdateOutTag:OutputTag[java.util.Map[String, AnyRef]]= OutputTag[java.util.Map[String, AnyRef]]("post-publish-relation-update") val searchBaseUrl = config.getString("service.search.basePath") val lmsBaseUrl = config.getString("service.lms.basePath") @@ -77,7 +83,29 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val reserveDialCodeAPIPath = learningBaseUrl + "/content/v3/dialcode/reserve" val batchAddCertTemplateAPIPath = lmsBaseUrl + "/private/v1/course/batch/cert/template/add" + // QR Image Generator val QRImageGeneratorTopic: String = config.getString("kafka.qrimage.topic") val primaryCategories: util.List[String] = if (config.hasPath("dialcode.linkable.primaryCategory")) config.getStringList("dialcode.linkable.primaryCategory") else util.Arrays.asList("Course") //List[String]("Course") + + val contentServiceBase: String = config.getString("content.basePath") + val contentReadURL = contentServiceBase+ "/content/v3/read/" + + val contentHierarchyTable: String = "content_hierarchy" + val identifier: String = "identifier" + val Hierarchy: String = "hierarchy" + val children: String = "children" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val course: String = "Course" + val parentCollections: String="parentCollections" + + val contentSystemUpdatePath = learningBaseUrl + "/system/v3/content/update/" + val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val name: String = "name" + + val contentCacheStore: Int = 0 + + } diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala index 3d19d7ab9..bd88ec8fe 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala @@ -47,6 +47,9 @@ class PostPublishProcessorStreamTask(config: PostPublishProcessorConfig, kafkaCo .name("dialcode-link-process").uid("dialcode-link-process").setParallelism(config.linkDialCodeParallelism) linkDialCodeStream.getSideOutput(config.generateQRImageOutTag).addSink(kafkaConnector.kafkaStringSink(config.QRImageGeneratorTopic)) + + processStreamTask.getSideOutput(config.postPublishRelationUpdateOutTag).process(new PostPublishRelationUpdaterFunction(config, httpUtil)) + .name("post-publish-relation-update-process").uid("post-publish-relation-update-process").setParallelism(config.postPublishRelationUpdateParallelism) env.execute(config.jobName) } } From 9571f717931aa040ceb3ce96aa21bdba14aff6ff Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Mon, 25 Sep 2023 16:59:26 +0530 Subject: [PATCH 11/88] Dev #104439 - Create new Flink job to validate the program certificate event generation -pom.xml changes for the job-config --- credential-generator/program-cert-pre-processor/pom.xml | 3 +-- .../programcert/task/ProgramCertPreProcessorConfig.scala | 4 ++-- jobs-distribution/pom.xml | 6 ++++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/pom.xml b/credential-generator/program-cert-pre-processor/pom.xml index 380842673..dfbaca15c 100644 --- a/credential-generator/program-cert-pre-processor/pom.xml +++ b/credential-generator/program-cert-pre-processor/pom.xml @@ -7,9 +7,8 @@ credential-generator 1.0 - org.sunbird program-cert-pre-processor - 1.0-SNAPSHOT + 1.0.0 jar diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index dc7d5f467..b518b1b34 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -8,7 +8,7 @@ import org.sunbird.job.BaseJobConfig import java.util -class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJobConfig(config, "collection-cert-pre-processor") { +class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJobConfig(config, "program-cert-pre-processor") { implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) @@ -21,7 +21,7 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob //kafka config val kafkaInputTopic: String = config.getString("kafka.input.topic") val kafkaOutputTopic: String = config.getString("kafka.output.topic") - val certificatePreProcessorConsumer: String = "collection-cert-pre-processor-consumer" + val certificatePreProcessorConsumer: String = "program-cert-pre-processor-consumer" val generateCertificateProducer = "generate-certificate-sink" override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") val generateCertificateParallelism:Int = config.getInt("task.generate_certificate.parallelism") diff --git a/jobs-distribution/pom.xml b/jobs-distribution/pom.xml index bbc03d9b5..cce8b2ecb 100644 --- a/jobs-distribution/pom.xml +++ b/jobs-distribution/pom.xml @@ -122,6 +122,12 @@ 1.0.0 jar + + org.sunbird + program-cert-pre-processor + 1.0.0 + jar + From dd7ad934bbb08b76d93a881897ca26511c6273df Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 26 Sep 2023 13:07:40 +0530 Subject: [PATCH 12/88] Updated config and added logs --- .../org/sunbird/job/certgen/functions/NotifierFunction.scala | 3 ++- .../job/postpublish/task/PostPublishProcessorConfig.scala | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala index 9711492cf..44d5e96d5 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala @@ -58,6 +58,7 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) { logger.info("notification template is present in the cert-templates object {}", certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) + logger.info("Sending notification email. URL: {}", url) val notifyTemplate = getNotifyTemplateFromRes(certTemplate.get(metaData.templateId)) val ratingUrl = config.domainUrl + config.ratingMidPoint + metaData.courseId + config.ratingEndPoint + metaData.batchId val request = mutable.Map[String, AnyRef]("request" -> (notifyTemplate ++ mutable.Map[String, AnyRef]( @@ -67,7 +68,7 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ config.recipientUserIds -> List[String](metaData.userId), config.ratingPageUrl -> ratingUrl, config.body -> "email body"))) - + logger.info("request body created for notification: {}", ScalaJsonUtil.serialize(request)) val response = httpUtil.post(url, ScalaJsonUtil.serialize(request)) if (response.status == 200) { metrics.incCounter(config.notifiedUserCount) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala index cbca6e353..fa8acd1b4 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala @@ -88,7 +88,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val QRImageGeneratorTopic: String = config.getString("kafka.qrimage.topic") val primaryCategories: util.List[String] = if (config.hasPath("dialcode.linkable.primaryCategory")) config.getStringList("dialcode.linkable.primaryCategory") else util.Arrays.asList("Course") //List[String]("Course") - val contentServiceBase: String = config.getString("content.basePath") + val contentServiceBase: String = config.getString("service.content.basePath") val contentReadURL = contentServiceBase+ "/content/v3/read/" val contentHierarchyTable: String = "content_hierarchy" From 12f85d76f519b3ab2ee94058a4eb0394927d3c65 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 26 Sep 2023 14:29:24 +0530 Subject: [PATCH 13/88] Added code for metrics list --- .../functions/PostPublishRelationUpdaterFunction.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index ec25e0a00..5072a986e 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -187,5 +187,9 @@ class PostPublishRelationUpdaterFunction( } } - override def metricsList(): List[String] = ??? + override def metricsList(): List[String] = { + List(config.postPublishRelationUpdateEventCount, + config.postPublishRelationUpdateSuccessCount, + config.postPublishRelationUpdateFailureCount) + } } \ No newline at end of file From 349cac28acb4f477ca9e99d58adb3c46b1032bdb Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Thu, 28 Sep 2023 09:01:20 +0530 Subject: [PATCH 14/88] Adding New Fields in the Email Template - CourseProviderName, CoursePosterImage (#28) 1. In the getCourseInfo() , in the contentReadApi added field posterImage to retrieve the posterImage details. Added the coursePosterImage in the Map 2. Added constants int he CollectionCerPrePorcessorConfig - posterImage. 3.CertificateGeneratorFunction - In the context.output added new fields in the NotificationMetaData call - coursePosterImage and providerName. 4.NotificationMetaData class added new fields - coursePosterImage and providerName. 5.Added constants int he CertificateGeneratorConfig - courseProvider, coursePosterImage. 6. Added empty string in the test class NotifierFunctionTest --- .../functions/IssueCertificateHelper.scala | 16 +++++++++++----- .../task/CollectionCertPreProcessorConfig.scala | 1 + .../functions/CertificateGeneratorFunction.scala | 2 +- .../job/certgen/functions/NotifierFunction.scala | 8 ++++++-- .../task/CertificateGeneratorConfig.scala | 5 ++++- .../job/certgen/spec/NotifierFunctionTest.scala | 2 +- 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index debd031de..cb9f5045b 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -3,7 +3,9 @@ package org.sunbird.job.collectioncert.functions import java.text.SimpleDateFormat import com.datastax.driver.core.querybuilder.QueryBuilder import com.datastax.driver.core.{Row, TypeTokens} +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper import org.slf4j.LoggerFactory import org.sunbird.job.Metrics import org.sunbird.job.cache.DataCache @@ -16,7 +18,6 @@ import scala.collection.JavaConverters._ trait IssueCertificateHelper { private[this] val logger = LoggerFactory.getLogger(classOf[CollectionCertPreProcessorFn]) - def issueCertificate(event:Event, template: Map[String, String])(cassandraUtil: CassandraUtil, cache:DataCache, contentCache: DataCache, metrics: Metrics, config: CollectionCertPreProcessorConfig, httpUtil: HttpUtil): String = { //validCriteria logger.info("issueCertificate i/p event =>"+event) @@ -233,7 +234,8 @@ trait IssueCertificateHelper { "providerName" -> providerName, "tag" -> event.batchId, "primaryCategory" -> courseInfo.getOrElse("primaryCategory", "").asInstanceOf[String], - "parentCollections" -> courseInfo.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + "parentCollections" -> courseInfo.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]], + "coursePosterImage" -> courseInfo.getOrElse("coursePosterImage", "").asInstanceOf[String], ) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) @@ -259,26 +261,30 @@ trait IssueCertificateHelper { def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): Map[String, Any] = { val courseMetadata = cache.getWithRetry(courseId) val courseInfoMap = if (null == courseMetadata || courseMetadata.isEmpty) { - val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory" + val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory,posterImage" val response = getAPICall(url, "content")(config, httpUtil, metrics) val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') + val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] Map( "courseId" -> courseId, "courseName" -> courseName, "parentCollections" -> parentCollections, - "primaryCategory" -> primaryCategory + "primaryCategory" -> primaryCategory, + "coursePosterImage" -> posterImage, ) } else { val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') val parentCollections = courseMetadata.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') Map( "courseId" -> courseId, "courseName" -> courseName, "parentCollections" -> parentCollections, - "primaryCategory" -> primaryCategory + "primaryCategory" -> primaryCategory, + "coursePosterImage" -> posterImage, ) } courseInfoMap diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala index 36eccf4a7..429c79d99 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala @@ -81,5 +81,6 @@ class CollectionCertPreProcessorConfig(override val config: Config) extends Base val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false val primaryCategory: String = "primaryCategory" val parentCollections: String = "parentCollections" + val posterImage: String = "posterImage" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index 679e83c8e..33551a7c1 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -316,7 +316,7 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val audit = ScalaJsonUtil.serialize(certificateAuditEvent) context.output(config.auditEventOutputTag, audit) logger.info("pushAuditEvent: certificate audit event success {}", audit) - context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset)) + context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.eData.get("providerName").asInstanceOf[String],event.eData.get("coursePosterImage").asInstanceOf[String])) context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { metrics.incCounter(config.failedEventCount) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala index 44d5e96d5..3bcc16d07 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala @@ -20,7 +20,7 @@ import org.sunbird.job.{BaseProcessFunction, Metrics} import scala.collection.JavaConverters._ import scala.collection.mutable -case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long) +case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long, courseProvider: String, coursePosterImage:String) class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[NotificationMetaData, String](config) { @@ -67,7 +67,11 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ config.heldDate -> dateFormatter.format(metaData.issuedOn), config.recipientUserIds -> List[String](metaData.userId), config.ratingPageUrl -> ratingUrl, - config.body -> "email body"))) + config.body -> "email body", + config.courseName -> metaData.courseName, + config.courseProvider -> metaData.courseProvider, + config.coursePosterImage -> metaData.coursePosterImage + ))) logger.info("request body created for notification: {}", ScalaJsonUtil.serialize(request)) val response = httpUtil.post(url, ScalaJsonUtil.serialize(request)) if (response.status == 200) { diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index 2328eeb9d..647bb6341 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -169,5 +169,8 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val generateProgramCertificateOutputTag: OutputTag[String] = OutputTag[String](generateProgramCertificateOutputTagName) val generateProgramCertificateProducer = "generate-program-certificate-sink" - + + val courseProvider: String ="courseProvider" + val coursePosterImage :String ="coursePosterImage" + } diff --git a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala index 2cd445226..54bbece4b 100755 --- a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala +++ b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala @@ -64,7 +64,7 @@ class NotifierFunctionTest extends BaseTestSpec { "NotifierFunction " should "should send notify user" in { implicit val notificationMetaTypeInfo: TypeInformation[NotificationMetaData] = TypeExtractor.getForClass(classOf[NotificationMetaData]) - new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0), null, metrics) + new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0,"",""), null, metrics) metrics.get(s"${notifierConfig.courseBatchdbReadCount}") should be(1) metrics.get(s"${notifierConfig.notifiedUserCount}") should be(1) metrics.get(s"${notifierConfig.skipNotifyUserCount}") should be(0) From 92d511f892866aefc1ac17709cb79eef233a2e64 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Sat, 30 Sep 2023 15:55:33 +0530 Subject: [PATCH 15/88] New Email Template - endpoint added for email notification (#31) New Email Template - endpoint added for email notification --- .../org/sunbird/job/certgen/functions/NotifierFunction.scala | 2 +- .../sunbird/job/certgen/task/CertificateGeneratorConfig.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala index 3bcc16d07..72c47c99a 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala @@ -52,7 +52,7 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ val row = getNotificationTemplates(primaryFields, metrics) val certTemplate = row.getMap(config.cert_templates, com.google.common.reflect.TypeToken.of(classOf[String]), TypeTokens.mapOf(classOf[String], classOf[String])) - val url = config.learnerServiceBaseUrl + config.notificationEndPoint + val url = config.learnerServiceBaseUrl + config.newEmailTemplateNotificationEndPoint if (certTemplate != null && StringUtils.isNotBlank(metaData.templateId) && certTemplate.containsKey(metaData.templateId) && certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) { diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index 647bb6341..ccc1f19aa 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -172,5 +172,5 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val courseProvider: String ="courseProvider" val coursePosterImage :String ="coursePosterImage" - + val newEmailTemplateNotificationEndPoint: String = "/v1/notification/email" } From f2c254dea7587991d718980d4852c7007e8ae783 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 2 Oct 2023 11:05:22 +0530 Subject: [PATCH 16/88] Fix for NPE --- .../functions/PostPublishEventRouter.scala | 2 +- .../PostPublishRelationUpdaterFunction.scala | 13 ++++++------- .../task/PostPublishProcessorConfig.scala | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala index 2c6cb4d50..4bc1a7614 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala @@ -55,7 +55,7 @@ class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpU context.output(config.linkDIALCodeOutTag, dialCodeDetails) //Process Post Publish Relation Update - context.output(config.postPublishRelationUpdateOutTag, null) + context.output(config.postPublishRelationUpdateOutTag, identifier) } else { metrics.incCounter(config.skippedEventCount) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 5072a986e..4e9527db3 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -31,7 +31,7 @@ class PostPublishRelationUpdaterFunction( config: PostPublishProcessorConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null -) extends BaseProcessFunction[java.util.Map[String, AnyRef], String](config) +) extends BaseProcessFunction[String, String](config) with PostPublishRelationUpdater { private[this] val logger = @@ -55,7 +55,7 @@ class PostPublishRelationUpdaterFunction( } private def postPublishRelationUpdate( - eData: util.Map[String, AnyRef] + identifier: String )(implicit config: PostPublishProcessorConfig, httpUtil: HttpUtil, @@ -63,7 +63,7 @@ class PostPublishRelationUpdaterFunction( metrics: Metrics ): Unit = { val programHierarchy = getProgramHierarchy( - eData.get("identifier").asInstanceOf[String] + identifier )(metrics, config, cache, httpUtil) if (programHierarchy.isEmpty) { logger.info( @@ -148,11 +148,10 @@ class PostPublishRelationUpdaterFunction( } override def processElement( - eData: java.util.Map[String, AnyRef], - context: ProcessFunction[java.util.Map[String, AnyRef], String]#Context, + identifier: String, + context: ProcessFunction[String, String]#Context, metrics: Metrics ): Unit = { - val identifier = eData.getOrDefault("identifier", "").asInstanceOf[String] val isValidProgram: Boolean = verifyPrimaryCategory(identifier)(metrics, config, httpUtil, cache) if (!isValidProgram) { @@ -161,7 +160,7 @@ class PostPublishRelationUpdaterFunction( "PostPublishRelationUpdaterFunction:: started for Content : " + identifier ) try { - postPublishRelationUpdate(eData)( + postPublishRelationUpdate(identifier)( config, httpUtil, cassandraUtil, diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala index fa8acd1b4..1db1b27d4 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala @@ -70,7 +70,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val shallowContentPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("shallow-copied-content-publish") val publishEventOutTag: OutputTag[String] = OutputTag[String]("content-publish-request") val generateQRImageOutTag: OutputTag[String] = OutputTag[String]("qr-image-generator-request") - val postPublishRelationUpdateOutTag:OutputTag[java.util.Map[String, AnyRef]]= OutputTag[java.util.Map[String, AnyRef]]("post-publish-relation-update") + val postPublishRelationUpdateOutTag:OutputTag[String]= OutputTag[String]("post-publish-relation-update") val searchBaseUrl = config.getString("service.search.basePath") val lmsBaseUrl = config.getString("service.lms.basePath") From a175d16cd23acf767b7c11a43378c4b53bde5265 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 2 Oct 2023 15:38:15 +0530 Subject: [PATCH 17/88] Fix for type cast issue --- .../sunbird/job/certgen/domain/Event.scala | 1 + .../CertificateGeneratorFunction.scala | 2 +- .../PostPublishRelationUpdaterFunction.scala | 85 +++++++++---------- .../helpers/PostPublishRelationUpdater.scala | 2 +- 4 files changed, 41 insertions(+), 49 deletions(-) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala index 7f7531e3a..c7259302b 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala @@ -64,4 +64,5 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + def coursePosterImage: String = readOrDefault[String]("edata.coursePosterImage", "") } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index 33551a7c1..bc69b8581 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -316,7 +316,7 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val audit = ScalaJsonUtil.serialize(certificateAuditEvent) context.output(config.auditEventOutputTag, audit) logger.info("pushAuditEvent: certificate audit event success {}", audit) - context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.eData.get("providerName").asInstanceOf[String],event.eData.get("coursePosterImage").asInstanceOf[String])) + context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { metrics.incCounter(config.failedEventCount) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 4e9527db3..a2e71bd65 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -20,8 +20,8 @@ import org.sunbird.job.{BaseProcessFunction, Metrics} import java.time.format.DateTimeFormatter import java.time.{ZoneId, ZonedDateTime} -import java.util import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} import scala.collection.mutable.ListBuffer /** @author @@ -72,58 +72,49 @@ class PostPublishRelationUpdaterFunction( return } - val childCourseList = programHierarchy - .get(config.children) - .asInstanceOf[List[Map[String, AnyRef]]] - for (childNode <- childCourseList) { + val childrenList = programHierarchy.get(config.children).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] + for (childNode <- childrenList) { val primaryCategory: String = childNode.get(config.primaryCategory).asInstanceOf[String] - if (primaryCategory.equalsIgnoreCase("Course")) { - val contentObj: java.util.Map[String, AnyRef] = getCourseInfo( - childNode.get("identifier").asInstanceOf[String] - )(metrics, config, cache, httpUtil) - val parentCollections: ListBuffer[String] = contentObj - .getOrDefault(config.parentCollections, ListBuffer.empty[String]) - .asInstanceOf[ListBuffer[String]] - val versionKey: String = - contentObj.get(config.versionKey).asInstanceOf[String] - val identifier: String = - childNode.get("identifier").asInstanceOf[String] - if (parentCollections.isEmpty) { - parentCollections += identifier - } else { - if (!parentCollections.contains(identifier)) { + val identifier: String = childNode.get("identifier").asInstanceOf[String] + if (primaryCategory.equalsIgnoreCase("Course")) { + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) + val parentCollections: ListBuffer[String] = contentObj.getOrDefault(config.parentCollections, ListBuffer.empty[String]).asInstanceOf[ListBuffer[String]] + val versionKey: String = contentObj.get(config.versionKey).asInstanceOf[String] + if (parentCollections.isEmpty) { parentCollections += identifier + } else { + if (!parentCollections.contains(identifier)) { + parentCollections += identifier + } } - } - val requestData: Map[String, Any] = Map( - "request" -> Map( - "content" -> Map( - "versionKey" -> versionKey, - "parentCollections" -> parentCollections.toList - ) + val requestData: Map[String, Any] = Map( + "request" -> Map( + "content" -> Map( + "versionKey" -> versionKey, + "parentCollections" -> parentCollections.toList + ) + )) + val jsonString: String = JSONUtil.serialize(requestData) + val patchRequest = new HttpPatch( + config.contentSystemUpdatePath + identifier ) - ) - val jsonString: String = JSONUtil.serialize(requestData) - val patchRequest = new HttpPatch( - config.contentSystemUpdatePath + identifier - ) - patchRequest.setEntity( - new StringEntity(jsonString, ContentType.APPLICATION_JSON) - ) - val httpClient = HttpClients.createDefault() - val response: HttpResponse = httpClient.execute(patchRequest) - val statusLine: StatusLine = response.getStatusLine - val statusCode: Int = statusLine.getStatusCode - if (statusCode == 200) { - logger.info("Processed the request.") - } else { - logger.error( - "Received error response for system update API. Response: " + JSONUtil - .serialize(response) + patchRequest.setEntity( + new StringEntity(jsonString, ContentType.APPLICATION_JSON) ) + val httpClient = HttpClients.createDefault() + val response: HttpResponse = httpClient.execute(patchRequest) + val statusLine: StatusLine = response.getStatusLine + val statusCode: Int = statusLine.getStatusCode + if (statusCode == 200) { + logger.info("Processed the request.") + } else { + logger.error( + "Received error response for system update API. Response: " + JSONUtil + .serialize(response) + ) + } } - } } } @@ -154,7 +145,7 @@ class PostPublishRelationUpdaterFunction( ): Unit = { val isValidProgram: Boolean = verifyPrimaryCategory(identifier)(metrics, config, httpUtil, cache) - if (!isValidProgram) { + if (isValidProgram) { metrics.incCounter(config.postPublishRelationUpdateEventCount) logger.info( "PostPublishRelationUpdaterFunction:: started for Content : " + identifier diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 1abaa7fe1..db5c42602 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -30,7 +30,7 @@ trait PostPublishRelationUpdater { var isValidProgram = false val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) - if (contentObj.isEmpty) { + if (!contentObj.isEmpty) { contentObj.get("primaryCategory") match { case Some("Program" | "Curated Program" | "Blended Program") => isValidProgram = true From 1ce7d1022803007d1441ccb1d7d1550df3f90734 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Mon, 2 Oct 2023 16:28:05 +0530 Subject: [PATCH 18/88] Fix for cast when matching string --- .../postpublish/helpers/PostPublishRelationUpdater.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index db5c42602..e697aa624 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -31,9 +31,12 @@ trait PostPublishRelationUpdater { val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) if (!contentObj.isEmpty) { - contentObj.get("primaryCategory") match { - case Some("Program" | "Curated Program" | "Blended Program") => - isValidProgram = true + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory == "Program" + || primaryCategory == "Curated Program" + || primaryCategory == "Blended Program")) { + isValidProgram = true } } isValidProgram From ddebd30aeadd04398b68154fa4e6e5a555cc46de Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Mon, 2 Oct 2023 18:26:20 +0530 Subject: [PATCH 19/88] Fixed for type casting --- .../functions/PostPublishRelationUpdaterFunction.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index a2e71bd65..64340dd56 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -79,20 +79,20 @@ class PostPublishRelationUpdaterFunction( val identifier: String = childNode.get("identifier").asInstanceOf[String] if (primaryCategory.equalsIgnoreCase("Course")) { val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) - val parentCollections: ListBuffer[String] = contentObj.getOrDefault(config.parentCollections, ListBuffer.empty[String]).asInstanceOf[ListBuffer[String]] + val parentCollections: java.util.ArrayList[String] = contentObj.getOrDefault(config.parentCollections, new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] val versionKey: String = contentObj.get(config.versionKey).asInstanceOf[String] if (parentCollections.isEmpty) { - parentCollections += identifier + parentCollections.add(identifier) } else { if (!parentCollections.contains(identifier)) { - parentCollections += identifier + parentCollections.add(identifier) } } val requestData: Map[String, Any] = Map( "request" -> Map( "content" -> Map( "versionKey" -> versionKey, - "parentCollections" -> parentCollections.toList + "parentCollections" -> parentCollections ) )) val jsonString: String = JSONUtil.serialize(requestData) From fa043c6ebbd99e23c7782e6f60d25d335ab277ea Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 07:17:51 +0530 Subject: [PATCH 20/88] Added logs --- .../job/postpublish/helpers/PostPublishRelationUpdater.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index e697aa624..b92f91c59 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -38,6 +38,9 @@ trait PostPublishRelationUpdater { || primaryCategory == "Blended Program")) { isValidProgram = true } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) } isValidProgram } From 1526299876ee19b6d841a1f5f01c810cc2c007bc Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 08:03:08 +0530 Subject: [PATCH 21/88] Handling null value for parentCollections --- .../PostPublishRelationUpdaterFunction.scala | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 64340dd56..771097ea5 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -79,20 +79,28 @@ class PostPublishRelationUpdaterFunction( val identifier: String = childNode.get("identifier").asInstanceOf[String] if (primaryCategory.equalsIgnoreCase("Course")) { val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) - val parentCollections: java.util.ArrayList[String] = contentObj.getOrDefault(config.parentCollections, new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] val versionKey: String = contentObj.get(config.versionKey).asInstanceOf[String] - if (parentCollections.isEmpty) { - parentCollections.add(identifier) - } else { - if (!parentCollections.contains(identifier)) { - parentCollections.add(identifier) + + // Use Option to safely handle null values + val parentCollections: List[String] = Option(contentObj.get(config.parentCollections)) + .collect { + case list: java.util.List[_] => + list.asInstanceOf[java.util.List[String]].asScala.toList } + .getOrElse(List.empty) + + // Update parentCollections if identifier is not present + val updatedParentCollections = if (!parentCollections.contains(identifier)) { + parentCollections :+ identifier + } else { + parentCollections } + val requestData: Map[String, Any] = Map( "request" -> Map( "content" -> Map( "versionKey" -> versionKey, - "parentCollections" -> parentCollections + "parentCollections" -> updatedParentCollections ) )) val jsonString: String = JSONUtil.serialize(requestData) From 0350b7e97d1cca97d56c9255cdae0fc248a92251 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 09:11:44 +0530 Subject: [PATCH 22/88] Added debug logs --- .../functions/PostPublishRelationUpdaterFunction.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 771097ea5..1374fafd6 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -104,6 +104,7 @@ class PostPublishRelationUpdaterFunction( ) )) val jsonString: String = JSONUtil.serialize(requestData) + logger.info("Calling content update with body: " + jsonString) val patchRequest = new HttpPatch( config.contentSystemUpdatePath + identifier ) From 8359a45d8174d364bcee3946136a07beaa4589e9 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 09:45:05 +0530 Subject: [PATCH 23/88] Handling null for versionKey --- .../functions/PostPublishRelationUpdaterFunction.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 1374fafd6..ac3787df6 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -79,7 +79,12 @@ class PostPublishRelationUpdaterFunction( val identifier: String = childNode.get("identifier").asInstanceOf[String] if (primaryCategory.equalsIgnoreCase("Course")) { val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) - val versionKey: String = contentObj.get(config.versionKey).asInstanceOf[String] + var versionKey: String = "" + val versionKeyOption: Option[String] = Option(contentObj.get(config.versionKey)).map(_.asInstanceOf[String]) + if (versionKeyOption.isDefined) { + versionKey = versionKeyOption.get + } + logger.info("Current versionKey is: " + versionKey) // Use Option to safely handle null values val parentCollections: List[String] = Option(contentObj.get(config.parentCollections)) From f8b531b2e76c45f5dfc7d2b83a2db9cb0dc183e5 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 10:43:39 +0530 Subject: [PATCH 24/88] Added versionKey --- .../helpers/PostPublishRelationUpdater.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index b92f91c59..9db306fc9 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -66,6 +66,11 @@ trait PostPublishRelationUpdater { response.getOrElse(config.primaryCategory, "").asInstanceOf[String] ) .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + response.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') val parentCollections = response .getOrElse("parentCollections", List.empty[String]) .asInstanceOf[List[String]] @@ -75,6 +80,7 @@ trait PostPublishRelationUpdater { courseInfoMap.put("courseName", courseName) courseInfoMap.put("parentCollections", parentCollections) courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) courseInfoMap } else { val courseName = StringContext @@ -89,6 +95,11 @@ trait PostPublishRelationUpdater { .asInstanceOf[String] ) .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') val parentCollections = courseMetadata .getOrElse("parentCollections", List.empty[String]) .asInstanceOf[List[String]] @@ -98,6 +109,7 @@ trait PostPublishRelationUpdater { courseInfoMap.put("courseName", courseName) courseInfoMap.put("parentCollections", parentCollections) courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) courseInfoMap } From f0b4fe2f1ad591480ced3b78135d115b614a4e7c Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 11:13:30 +0530 Subject: [PATCH 25/88] Added logs --- .../job/aggregate/functions/ActivityAggregatesFunction.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index d80e12e7d..136a0da0d 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.aggregate.domain.{UserContentConsumption, _} import org.sunbird.job.aggregate.task.ActivityAggregateUpdaterConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} import org.sunbird.job.{Metrics, WindowBaseProcessFunction} import scala.collection.JavaConverters._ @@ -57,7 +57,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti events: Iterable[Map[String, AnyRef]], metrics: Metrics): Unit = { - logger.debug("Input Events Size: " + events.toList.size) + logger.info("Input Events : " + JSONUtil.serialize(events.toList)) val inputUserConsumptionList: List[UserContentConsumption] = events .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { From cc0fc2f384d98bca8407452afbe790c4bd759873 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 13:01:03 +0530 Subject: [PATCH 26/88] Adding proper identifier in parentCollections --- .../functions/PostPublishRelationUpdaterFunction.scala | 6 +++--- .../postpublish/helpers/PostPublishRelationUpdater.scala | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index ac3787df6..6d8ce71aa 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -76,9 +76,9 @@ class PostPublishRelationUpdaterFunction( for (childNode <- childrenList) { val primaryCategory: String = childNode.get(config.primaryCategory).asInstanceOf[String] - val identifier: String = childNode.get("identifier").asInstanceOf[String] + val childId: String = childNode.get("identifier").asInstanceOf[String] if (primaryCategory.equalsIgnoreCase("Course")) { - val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(identifier)(metrics, config, cache, httpUtil) + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(childId)(metrics, config, cache, httpUtil) var versionKey: String = "" val versionKeyOption: Option[String] = Option(contentObj.get(config.versionKey)).map(_.asInstanceOf[String]) if (versionKeyOption.isDefined) { @@ -111,7 +111,7 @@ class PostPublishRelationUpdaterFunction( val jsonString: String = JSONUtil.serialize(requestData) logger.info("Calling content update with body: " + jsonString) val patchRequest = new HttpPatch( - config.contentSystemUpdatePath + identifier + config.contentSystemUpdatePath + childId ) patchRequest.setEntity( new StringEntity(jsonString, ContentType.APPLICATION_JSON) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 9db306fc9..b16bea0b5 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -56,6 +56,7 @@ trait PostPublishRelationUpdater { val url = config.contentReadURL + "/" + courseId + "?fields=identifier,name,versionKey,parentCollections,primaryCategory" val response = getAPICall(url, "content")(config, httpUtil, metrics) + logger.info("Content read response" + JSONUtil.serialize(response)) val courseName = StringContext .processEscapes( response.getOrElse(config.name, "").asInstanceOf[String] @@ -80,7 +81,7 @@ trait PostPublishRelationUpdater { courseInfoMap.put("courseName", courseName) courseInfoMap.put("parentCollections", parentCollections) courseInfoMap.put("primaryCategory", primaryCategory) - courseInfoMap.put("versionKey", versionKey) + courseInfoMap.put(config.versionKey, versionKey) courseInfoMap } else { val courseName = StringContext @@ -109,7 +110,7 @@ trait PostPublishRelationUpdater { courseInfoMap.put("courseName", courseName) courseInfoMap.put("parentCollections", parentCollections) courseInfoMap.put("primaryCategory", primaryCategory) - courseInfoMap.put("versionKey", versionKey) + courseInfoMap.put(config.versionKey, versionKey) courseInfoMap } From 6fc810d36e5395d5c0120b012ef7e2f16eb97f52 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 14:24:38 +0530 Subject: [PATCH 27/88] Added logs --- .../PostPublishRelationUpdaterFunction.scala | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala index 6d8ce71aa..a2c96daee 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -74,17 +74,12 @@ class PostPublishRelationUpdaterFunction( val childrenList = programHierarchy.get(config.children).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] for (childNode <- childrenList) { - val primaryCategory: String = - childNode.get(config.primaryCategory).asInstanceOf[String] + val primaryCategory: String = childNode.get(config.primaryCategory).asInstanceOf[String] val childId: String = childNode.get("identifier").asInstanceOf[String] if (primaryCategory.equalsIgnoreCase("Course")) { val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(childId)(metrics, config, cache, httpUtil) - var versionKey: String = "" - val versionKeyOption: Option[String] = Option(contentObj.get(config.versionKey)).map(_.asInstanceOf[String]) - if (versionKeyOption.isDefined) { - versionKey = versionKeyOption.get - } - logger.info("Current versionKey is: " + versionKey) + var versionKey: String = contentObj.getOrDefault(config.versionKey, "").asInstanceOf[String] + logger.info("Child Course Id: " + childId + ", Info: " + JSONUtil.serialize(contentObj)) // Use Option to safely handle null values val parentCollections: List[String] = Option(contentObj.get(config.parentCollections)) From ca5ac4e0ab286427f61f7a9bcb9af54da115a011 Mon Sep 17 00:00:00 2001 From: tarentomaheshvakkund <139739142+tarentomaheshvakkund@users.noreply.github.com> Date: Tue, 3 Oct 2023 15:45:34 +0530 Subject: [PATCH 28/88] Dev #106998-Need to enhance activity aggregate flink job to skip events for program type courses 1. Added a method verifyPrimaryCategory() to check if the primary category is blended program, curated program or program and skip the event accordingly. --- .../ActivityAggregatesFunction.scala | 134 +++++++++++++++++- .../task/ActivityAggregateUpdaterConfig.scala | 9 ++ 2 files changed, 140 insertions(+), 3 deletions(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index 136a0da0d..3e36375c9 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -2,7 +2,6 @@ package org.sunbird.job.aggregate.functions import java.lang.reflect.Type import java.util.concurrent.TimeUnit - import com.datastax.driver.core.Row import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} import com.google.gson.Gson @@ -19,10 +18,11 @@ import org.slf4j.LoggerFactory import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.aggregate.domain.{UserContentConsumption, _} import org.sunbird.job.aggregate.task.ActivityAggregateUpdaterConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, ScalaJsonUtil} import org.sunbird.job.{Metrics, WindowBaseProcessFunction} import scala.collection.JavaConverters._ +import scala.language.postfixOps class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) (implicit val stringTypeInfo: TypeInformation[String]) @@ -59,6 +59,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti logger.info("Input Events : " + JSONUtil.serialize(events.toList)) val inputUserConsumptionList: List[UserContentConsumption] = events + .filter(event=> verifyPrimaryCategory(event.get(config.courseId).asInstanceOf[String])(metrics, config, httpUtil, cache)) .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { metrics.incCounter(config.processedEnrolmentCount) @@ -444,5 +445,132 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti dbStatus } else cacheStatus } -} + def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: ActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (!contentObj.isEmpty) { + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory != "Program" + || primaryCategory != "Curated Program" + || primaryCategory != "Blended Program")) { + isValidProgram = true + } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) + } + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + "/" + courseId + "?fields=identifier,name,versionKey,parentCollections,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + response.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } +} diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala index f36edaf4c..1e024f408 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala @@ -132,4 +132,13 @@ class ActivityAggregateUpdaterConfig(override val config: Config) extends BaseJo val searchServiceBasePath: String = config.getString("service.search.basePath") val searchAPIURL = searchServiceBasePath + "/v3/search" + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase + "/content/v3/read/" + val name: String = "name" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val skippedEventCount = "skipped-events-count" + } From 285b3c60e38217cd18aed582d2e58647b004b33c Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 15:49:48 +0530 Subject: [PATCH 29/88] Added logs --- .../job/postpublish/helpers/PostPublishRelationUpdater.scala | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index b16bea0b5..8e8b047e6 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -84,6 +84,11 @@ trait PostPublishRelationUpdater { courseInfoMap.put(config.versionKey, versionKey) courseInfoMap } else { + logger.info("Content read response from cache " + JSONUtil.serialize(courseMetadata)) + val name = courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + val category = courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String] + val version = courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] + logger.info("name: " + name, ", category: " + category + ", version: " + version) val courseName = StringContext .processEscapes( courseMetadata.getOrElse(config.name, "").asInstanceOf[String] From 07a797bbc97e7feca7834277de3daa2fccdd366f Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 16:42:07 +0530 Subject: [PATCH 30/88] Updated code to read cache values --- .../helpers/PostPublishRelationUpdater.scala | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 8e8b047e6..8191f90df 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -88,34 +88,17 @@ trait PostPublishRelationUpdater { val name = courseMetadata.getOrElse(config.name, "").asInstanceOf[String] val category = courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String] val version = courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] - logger.info("name: " + name, ", category: " + category + ", version: " + version) - val courseName = StringContext - .processEscapes( - courseMetadata.getOrElse(config.name, "").asInstanceOf[String] - ) - .filter(_ >= ' ') - val primaryCategory = StringContext - .processEscapes( - courseMetadata - .getOrElse(config.primaryCategory, "") - .asInstanceOf[String] - ) - .filter(_ >= ' ') - val versionKey = StringContext - .processEscapes( - courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] - ) - .filter(_ >= ' ') + logger.info("name: " + name + ", category: " + category + ", version: " + version) val parentCollections = courseMetadata .getOrElse("parentCollections", List.empty[String]) .asInstanceOf[List[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) - courseInfoMap.put("courseName", courseName) + courseInfoMap.put("courseName", name) courseInfoMap.put("parentCollections", parentCollections) - courseInfoMap.put("primaryCategory", primaryCategory) - courseInfoMap.put(config.versionKey, versionKey) + courseInfoMap.put("primaryCategory", category) + courseInfoMap.put(config.versionKey, version) courseInfoMap } From 5b904c22b15be6b21c9f8f69e58d45754248127a Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 17:36:00 +0530 Subject: [PATCH 31/88] Updated key when fetching from redis --- .../postpublish/helpers/PostPublishRelationUpdater.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 8191f90df..54db9a89e 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -86,11 +86,11 @@ trait PostPublishRelationUpdater { } else { logger.info("Content read response from cache " + JSONUtil.serialize(courseMetadata)) val name = courseMetadata.getOrElse(config.name, "").asInstanceOf[String] - val category = courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String] - val version = courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] + val category = courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String] + val version = courseMetadata.getOrElse("versionkey", "").asInstanceOf[String] logger.info("name: " + name + ", category: " + category + ", version: " + version) val parentCollections = courseMetadata - .getOrElse("parentCollections", List.empty[String]) + .getOrElse("parentcollections", List.empty[String]) .asInstanceOf[List[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() From b9ef9891bbbd531f75b7192b76162b9a74c6176c Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 17:40:39 +0530 Subject: [PATCH 32/88] Updated key when fetching from redis --- .../collectioncert/functions/IssueCertificateHelper.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index cb9f5045b..9c4dde899 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -276,9 +276,9 @@ trait IssueCertificateHelper { ) } else { val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') - val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') - val parentCollections = courseMetadata.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] - val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = courseMetadata.getOrElse("parentcollections", List.empty[String]).asInstanceOf[List[String]] + val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse("posterimage", "").asInstanceOf[String]).filter(_ >= ' ') Map( "courseId" -> courseId, "courseName" -> courseName, From 8c31a74313fad9c89ae09f68857b4062f7b19625 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 3 Oct 2023 18:29:38 +0530 Subject: [PATCH 33/88] Fixed type cast issue --- .../collectioncert/functions/IssueCertificateHelper.scala | 4 ++-- .../postpublish/helpers/PostPublishRelationUpdater.scala | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 9c4dde899..794e43da5 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -266,7 +266,7 @@ trait IssueCertificateHelper { val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') - val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + val parentCollections = response.getOrElse("parentCollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] Map( "courseId" -> courseId, "courseName" -> courseName, @@ -277,7 +277,7 @@ trait IssueCertificateHelper { } else { val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') - val parentCollections = courseMetadata.getOrElse("parentcollections", List.empty[String]).asInstanceOf[List[String]] + val parentCollections = courseMetadata.getOrElse("parentcollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse("posterimage", "").asInstanceOf[String]).filter(_ >= ' ') Map( "courseId" -> courseId, diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 54db9a89e..7badd99fc 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -73,8 +73,8 @@ trait PostPublishRelationUpdater { ) .filter(_ >= ' ') val parentCollections = response - .getOrElse("parentCollections", List.empty[String]) - .asInstanceOf[List[String]] + .getOrElse("parentCollections", new java.util.ArrayList()) + .asInstanceOf[java.util.ArrayList[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) @@ -90,8 +90,8 @@ trait PostPublishRelationUpdater { val version = courseMetadata.getOrElse("versionkey", "").asInstanceOf[String] logger.info("name: " + name + ", category: " + category + ", version: " + version) val parentCollections = courseMetadata - .getOrElse("parentcollections", List.empty[String]) - .asInstanceOf[List[String]] + .getOrElse("parentcollections", new java.util.ArrayList()) + .asInstanceOf[java.util.ArrayList[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) From 3423a582d51cc5875d7d2165cd5733cf77b02d27 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 3 Oct 2023 19:16:57 +0530 Subject: [PATCH 34/88] Dev #106996 - Need to create new Flink job to process activity aggregate events for Program --- jobs-distribution/pom.xml | 6 + pom.xml | 1 + program-activity-aggregate-updater/pom.xml | 242 +++++++ .../src/main/resources/log4j.properties | 11 + .../program-activity-aggregate-updater.conf | 54 ++ .../programaggregate/common/DeDupHelper.scala | 12 + .../job/programaggregate/domain/Models.scala | 52 ++ .../ProgramActivityAggregatesFunction.scala | 596 ++++++++++++++++++ .../ProgramProgressCompleteFunction.scala | 141 +++++ .../ProgramProgressUpdateFunction.scala | 96 +++ ...rogramActivityAggregateUpdaterConfig.scala | 149 +++++ ...amActivityAggregateUpdaterStreamTask.scala | 71 +++ .../src/test/resources/logback-test.xml | 18 + .../src/test/resources/test.conf | 53 ++ .../src/test/resources/test.cql | 79 +++ .../sunbird/job/fixture/EventFixture.scala | 194 ++++++ .../spec/BaseActivityAggregateTestSpec.scala | 59 ++ ...ActivityAggregateUpdaterTaskTestSpec.scala | 219 +++++++ 18 files changed, 2053 insertions(+) create mode 100644 program-activity-aggregate-updater/pom.xml create mode 100644 program-activity-aggregate-updater/src/main/resources/log4j.properties create mode 100644 program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala create mode 100644 program-activity-aggregate-updater/src/test/resources/logback-test.xml create mode 100644 program-activity-aggregate-updater/src/test/resources/test.conf create mode 100644 program-activity-aggregate-updater/src/test/resources/test.cql create mode 100644 program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala create mode 100644 program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala create mode 100644 program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala diff --git a/jobs-distribution/pom.xml b/jobs-distribution/pom.xml index cce8b2ecb..ca66efe24 100644 --- a/jobs-distribution/pom.xml +++ b/jobs-distribution/pom.xml @@ -128,6 +128,12 @@ 1.0.0 jar + + org.sunbird + program-activity-aggregate-updater + 1.0.0 + jar + diff --git a/pom.xml b/pom.xml index 84e84ec96..18fe6e0a3 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ audit-event-generator metrics-data-transformer qrcode-image-generator + program-activity-aggregate-updater diff --git a/program-activity-aggregate-updater/pom.xml b/program-activity-aggregate-updater/pom.xml new file mode 100644 index 000000000..167447f4c --- /dev/null +++ b/program-activity-aggregate-updater/pom.xml @@ -0,0 +1,242 @@ + + + 4.0.0 + + org.sunbird + knowledge-platform-jobs + 1.0 + + org.sunbird + program-activity-aggregate-updater + 1.0.0 + jar + program-activity-aggregate-updater + + Program Progress Computation + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.version} + ${flink.version} + provided + + + org.sunbird + jobs-core + 1.0.0 + + + joda-time + joda-time + 2.10.6 + + + com.twitter + storehaus-cache_${scala.version} + 0.15.0 + + + org.sunbird + jobs-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_${scala.version} + ${flink.version} + test + + + org.apache.flink + flink-runtime_${scala.version} + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + org.apache.flink + flink-streaming-java_${scala.version} + ${flink.version} + test + tests + + + org.scalatest + scalatest_${scala.version} + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + it.ozimov + embedded-redis + 0.7.1 + test + + + com.google.guava + guava + + + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.job.programaggregate.task.ActivityAggregateUpdaterStreamTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + 11 + 11 + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + dp-duplication-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + diff --git a/program-activity-aggregate-updater/src/main/resources/log4j.properties b/program-activity-aggregate-updater/src/main/resources/log4j.properties new file mode 100644 index 000000000..cacd49c79 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/resources/log4j.properties @@ -0,0 +1,11 @@ +# log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.file=course-metrics-updater.log +log4j.appender.file.append=true +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.MaxFileSize=256KB +log4j.appender.file.MaxBackupIndex=4 +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf new file mode 100644 index 000000000..9f8d79d13 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf @@ -0,0 +1,54 @@ +include "base-config.conf" + +kafka { + input.topic = "sunbirddev.coursebatch.job.request" + output.audit.topic = "sunbirddev.telemetry.raw" + output.failed.topic = "sunbirddev.activity.agg.failed" + output.certissue.topic = "sunbirddev.issue.certificate.request" + groupId = "sunbirddev-activity-aggregate-updater-group" +} + +task { + window.shards = 1 + consumer.parallelism = 1 + dedup.parallelism = 1 + activity.agg.parallelism = 1 + enrolment.complete.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + consumption.table = "user_content_consumption" + user_activity_agg.table = "user_activity_agg" + user_enrolments.table = "user_enrolments" +} + +redis { + database { + relationCache.id = 10 + } +} + +dedup-redis { + host = 11.2.4.22 + port = 6379 + database.index = 3 + database.expiry = 604800 +} + +threshold.batch.read.interval = 60 // In sec +threshold.batch.read.size = 1000 +threshold.batch.write.size = 10 + +activity { + module.aggs.enabled = true + input.dedup.enabled = true + filter.processed.enrolments = true + collection.status.cache.expiry = 3600 +} + +service { + search { + basePath = "http://11.2.6.6/search" + } +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala new file mode 100644 index 000000000..3bce467e3 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala @@ -0,0 +1,12 @@ +package org.sunbird.job.aggregate.common + +import java.security.MessageDigest + +object DeDupHelper { + + def getMessageId(collectionId: String, batchId: String, userId: String, contentId: String, status: Int): String = { + val key = Array(collectionId, batchId, userId, contentId, status).mkString("|") + MessageDigest.getInstance("MD5").digest(key.getBytes).map("%02X".format(_)).mkString; + } + +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala new file mode 100644 index 000000000..c694e5665 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala @@ -0,0 +1,52 @@ +package org.sunbird.job.programaggregate.domain + +import java.util +import java.util.{Date, UUID} + +import scala.collection.JavaConverters._ + + +case class ActorObject(id: String, `type`: String = "User") + +case class EventContext(channel: String = "in.sunbird", + env: String = "Course", + sid: String = UUID.randomUUID().toString, + did: String = UUID.randomUUID().toString, + pdata: util.Map[String, String] = Map("ver" -> "3.0", "id" -> "org.sunbird.learning.platform", "pid" -> "course-progress-updater").asJava, + cdata: Array[util.Map[String, String]]) + + +case class EventData(props: Array[String], `type`: String) + +case class EventObject(id: String, `type`: String, rollup: util.Map[String, String]) + +case class TelemetryEvent(actor: ActorObject, + eid: String = "AUDIT", + edata: EventData, + ver: String = "3.0", + syncts: Long = System.currentTimeMillis(), + ets: Long = System.currentTimeMillis(), + context: EventContext = EventContext( + cdata = Array[util.Map[String, String]]() + ), + mid: String = s"LP.AUDIT.${UUID.randomUUID().toString}", + `object`: EventObject, + tags: util.List[AnyRef] = new util.ArrayList[AnyRef]() + ) + +case class ContentStatus(contentId: String, status: Int = 0, completedCount: Int = 0, viewCount: Int = 1, fromInput: Boolean = true, eventsFor: List[String] = List()) + +case class UserContentConsumption(userId: String, batchId: String, courseId: String, contents: Map[String, ContentStatus]) + +case class UserActivityAgg(activity_type: String, + user_id: String, + activity_id: String, + context_id: String, + aggregates: Map[String, Double], + agg_last_updated: Map[String, Long] + ) + +case class CollectionProgress(userId: String, batchId: String, courseId: String, progress: Int, completedOn: Date, contentStatus: Map[String, Int], inputContents: List[String], completed: Boolean = false) + +case class UserEnrolmentAgg(activityAgg: UserActivityAgg, collectionProgress: Option[CollectionProgress] = None) + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala new file mode 100644 index 000000000..49e2ba5bb --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -0,0 +1,596 @@ +package org.sunbird.job.programaggregate.functions + +import java.lang.reflect.Type +import java.util.concurrent.TimeUnit +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.{Metrics, WindowBaseProcessFunction} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) + (implicit val stringTypeInfo: TypeInformation[String]) + extends WindowBaseProcessFunction[Map[String, AnyRef], String, Int](config) { + + val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramActivityAggregatesFunction]) + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() + + override def metricsList(): List[String] = { + List(config.failedEventCount, config.dbUpdateCount, config.dbReadCount, config.cacheHitCount, config.cacheMissCount, config.processedEnrolmentCount, config.retiredCCEventsCount) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + override def process(key: Int, + context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, + events: Iterable[Map[String, AnyRef]], + metrics: Metrics): Unit = { + + logger.debug("Input Events Size: " + events.toList.size) + + val eventsInfo = events.filter(event => getProgramEvent(event)(metrics, config, httpUtil, cache) != null) + val inputUserConsumptionList: List[UserContentConsumption] = eventsInfo + .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) + .values.map(value => { + metrics.incCounter(config.processedEnrolmentCount) + val batchId = value.head(config.batchId).toString + val userId = value.head(config.userId).toString + val courseId = value.head(config.courseId).toString + val userConsumedContents = value.head(config.contents).asInstanceOf[List[Map[String, AnyRef]]] + val enrichedContents = getContentStatusFromEvent(userConsumedContents) + UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) + }).toList + + if(inputUserConsumptionList.isEmpty) + return + // Fetch the content status from the table in batch format + val dbUserConsumption: Map[String, UserContentConsumption] = getContentStatusFromDB(events.toList, metrics) + + // Final User's ContentConsumption after merging with DB data. + // Here we have final viewcount, completedcount and identified the content which should generate AUDIT events for start and complete. + val finalUserConsumptionList = inputUserConsumptionList.map(inputData => { + val dbData = dbUserConsumption.getOrElse(getUCKey(inputData), UserContentConsumption(inputData.userId, inputData.batchId, inputData.courseId, Map())) + finalUserConsumption(inputData, dbData)(metrics) + }) + + // user_content_consumption update with viewcount and completedcout. + val userConsumptionQueries = finalUserConsumptionList.flatMap(userConsumption => getContentConsumptionQueries(userConsumption)) + updateDB(config.thresholdBatchWriteSize, userConsumptionQueries)(metrics) + + + val courseAggregations = finalUserConsumptionList.flatMap(userConsumption => { + + // Course Level Agg using the merged data of ContentConsumption per user, course and batch. + val optCourseAgg = courseActivityAgg(userConsumption, context)(metrics) + val courseAggs = if (optCourseAgg.nonEmpty) List(optCourseAgg.get) else List() + + // Identify the children of the course (only collections) for which aggregates computation required. + // Computation of aggregates using leafNodes (of the specific collection) and user completed contents. + // Here computing only "completedCount" aggregate. + if (config.moduleAggEnabled) { + val courseChildrenAggs = courseChildrenActivityAgg(userConsumption)(metrics) + courseAggs ++ courseChildrenAggs + } else courseAggs + }) + + // Saving all queries for course and it's children (only collection) aggregates. + val aggQueries = courseAggregations.map(agg => getUserAggQuery(agg.activityAgg)) + updateDB(config.thresholdBatchWriteSize, aggQueries)(metrics) + + // Saving enrolment completion data. + val collectionProgressList = courseAggregations.filter(agg => agg.collectionProgress.nonEmpty).map(agg => agg.collectionProgress.get) + + val collectionProgressUpdateList = collectionProgressList.filter(progress => !progress.completed) + context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) + + val collectionProgressCompleteList = collectionProgressList.filter(progress => progress.completed) + context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) + + // Content AUDIT Event generation and pushing to output tag. + finalUserConsumptionList.flatMap(userConsumption => contentAuditEvents(userConsumption)).foreach(event => context.output(config.auditEventOutputTag, gson.toJson(event))) + } + + /** + * Course Level Agg using the merged data of ContentConsumption per user, course and batch. + */ + def courseActivityAgg(userConsumption: UserContentConsumption, context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context)(implicit metrics: Metrics): Option[UserEnrolmentAgg] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + val key = s"$courseId:$courseId:${config.leafNodes}" + val leafNodes = readFromCache(key, metrics).distinct + if (leafNodes.isEmpty) { + logger.error(s"leaf nodes are not available for: $key") + context.output(config.failedEventOutputTag, gson.toJson(userConsumption)) + val status = getCollectionStatus(courseId) + if (StringUtils.equals("Retired", status)) { + metrics.incCounter(config.retiredCCEventsCount) + println(s"contents consumed from a retired collection: $courseId") + logger.warn(s"contents consumed from a retired collection: $courseId") + None + } else { + metrics.incCounter(config.failedEventCount) + val message = s"leaf nodes are not available for a published collection: $courseId" + logger.error(message) + throw new Exception(message) + } + } else { + val completedCount = leafNodes.intersect(userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct).size + val contentStatus = userConsumption.contents.map(cc => (cc._2.contentId, cc._2.status)).toMap + val inputContents = userConsumption.contents.filter(cc => cc._2.fromInput).keys.toList + val collectionProgress = if (completedCount >= leafNodes.size) { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, new java.util.Date(), contentStatus, inputContents, true)) + } else { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, null, contentStatus, inputContents)) + } + Option(UserEnrolmentAgg(UserActivityAgg("Course", userId, courseId, contextId, Map("completedCount" -> completedCount.toDouble), Map("completedCount" -> System.currentTimeMillis())), collectionProgress)) + } + } + + /** + * Identified the children of the course (only collections) for which aggregates computation required. + * Computation of aggregates using leafNodes (of the specific collection) and user completed contents. + * Here computing only "completedCount" aggregate. + */ + def courseChildrenActivityAgg(userConsumption: UserContentConsumption)(implicit metrics: Metrics): List[UserEnrolmentAgg] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + + // These are the child collections which require computation of aggregates - for this user. + val ancestors = userConsumption.contents.mapValues(content => { + val contentId = content.contentId + readFromCache(key = s"$courseId:$contentId:${config.ancestors}", metrics) + }).values.flatten.filter(a => !StringUtils.equals(a, courseId)).toList.distinct + + // LeafNodes of the identified child collections - for this user. + val collectionsWithLeafNodes = ancestors.map(unitId => { + (unitId, readFromCache(key = s"$courseId:$unitId:${config.leafNodes}", metrics).distinct) + }).toMap + + // Content completed - By this user. + val userCompletedContents = userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct + + // Child Collection UserAggregate list - for this user. + collectionsWithLeafNodes.map(e => { + val collectionId = e._1 + val leafNodes = e._2 + val completedCount = leafNodes.intersect(userCompletedContents).size + /* TODO - List + TODO 1. Generalise activityType from "Course" to "Collection". + TODO 2.Identify how to generate start and end event for CourseUnit. + */ + val activityAgg = UserActivityAgg("Course", userId, collectionId, contextId, Map("completedCount" -> completedCount), Map("completedCount" -> System.currentTimeMillis())) + UserEnrolmentAgg(activityAgg, None) + }).toList + } + + /** + * Generation of a "String" key for UserContentConsumption. + */ + def getUCKey(userConsumption: UserContentConsumption): String = { + userConsumption.userId + ":" + userConsumption.courseId + ":" + userConsumption.batchId + } + + /** + * Merging the Input and DB ContentStatus data of a User, Course and Batch (Enrolment) + * This is the critical part of the code. + */ + def finalUserConsumption(inputData: UserContentConsumption, dbData: UserContentConsumption)(implicit metrics: Metrics): UserContentConsumption = { + val dbContents = dbData.contents + val processedContents = inputData.contents.map { + case (contentId, inputCC) => { + // ContentStatus from DB. + val dbCC: ContentStatus = dbContents.getOrElse(contentId, ContentStatus(contentId, 0, 0, 0)) + val finalStatus = List(inputCC.status, dbCC.status).max // Final status is max of DB and Input ContentStatus. + val views = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { x.viewCount }) // View Count is sum of DB and Input ContentStatus. + val completion = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { x.completedCount }) // Completed Count is sum of DB and Input ContentStatus. + val eventsFor: List[String] = getEventActions(dbCC, inputCC) + // Merged ContentStatus. + (contentId, ContentStatus(contentId, finalStatus, completion, views, inputCC.fromInput, eventsFor)) + } + } + + val existingContents = processedContents.keys.toList + val remainingContents = dbData.contents.filterKeys(key => !existingContents.contains(key)) + val finalContentsMap = processedContents ++ remainingContents + UserContentConsumption(inputData.userId, inputData.batchId, inputData.courseId, finalContentsMap) + } + + /** + * This will identify whether this is the start or complete of the Content by User. + * + * @return List - Actions - "start" and "complete". + */ + def getEventActions(dbCC: ContentStatus, inputCC: ContentStatus): List[String] = { + val startAction = if (dbCC.viewCount == 0) List("start") else List() + val completeAction = if (dbCC.completedCount == 0 && inputCC.completedCount > 0) List(config.complete) else List() + startAction ::: completeAction + } + + /** + * Generic method to read data from DB (Cassandra). + * + * @return + */ + def readFromDB(columns: Map[String, AnyRef], keySpace: String, table: String, metrics: Metrics): List[Row] = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(keySpace, table). + where() + columns.map(col => { + col._2 match { + case value: List[Any] => + selectWhere.and(QueryBuilder.in(col._1, value.asJava)) + case _ => + selectWhere.and(QueryBuilder.eq(col._1, col._2)) + } + }) + metrics.incCounter(config.dbReadCount) + cassandraUtil.find(selectWhere.toString).asScala.toList + + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + val result = cassandraUtil.upsert(cqlBatch.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed: " + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } + + def readFromCache(key: String, metrics: Metrics): List[String] = { + metrics.incCounter(config.cacheHitCount) + val list = cache.getKeyMembers(key) + if (CollectionUtils.isEmpty(list)) { + metrics.incCounter(config.cacheMissCount) + logger.info("Redis cache (smembers) not available for key: " + key) + } + list.asScala.toList + } + + def getUserAggQuery(progress: UserActivityAgg): + Update.Where = { + QueryBuilder.update(config.dbKeyspace, config.dbUserActivityAggTable) + .`with`(QueryBuilder.putAll(config.aggregates, progress.aggregates.asJava)) + .and(QueryBuilder.putAll(config.aggLastUpdated, progress.agg_last_updated.asJava)) + .where(QueryBuilder.eq(config.activityId, progress.activity_id)) + .and(QueryBuilder.eq(config.activityType, progress.activity_type)) + .and(QueryBuilder.eq(config.contextId, progress.context_id)) + .and(QueryBuilder.eq(config.activityUser, progress.user_id)) + } + + /** + * Creates the cql query for content consumption table + */ + def getContentConsumptionQueries(userContentConsumption: UserContentConsumption): List[Update.Where] = { + userContentConsumption.contents.mapValues(content => { + QueryBuilder.update(config.dbKeyspace, config.dbUserContentConsumptionTable) + .`with`(QueryBuilder.set(config.viewcount, content.viewCount)) + .and(QueryBuilder.set(config.completedcount, content.completedCount)) + .where(QueryBuilder.eq(config.batchId.toLowerCase(), userContentConsumption.batchId)) + .and(QueryBuilder.eq(config.courseId.toLowerCase(), userContentConsumption.courseId)) + .and(QueryBuilder.eq(config.userId.toLowerCase(), userContentConsumption.userId)) + .and(QueryBuilder.eq(config.contentId.toLowerCase(), content.contentId)) + }).values.toList + } + + /** + * Method to get the content status object in map format ex: (do_5874308329084 -> 2, do_59485345435 -> 3) + * It always takes the highest precedence progress values for the contents ex: (do_5874308329084 -> 2, do_5874308329084 -> 1, do_59485345435 -> 3) => (do_5874308329084 -> 2, do_59485345435 -> 3) + * + * Ex: Map("C1"->2, "C2" ->1) + * + */ + def getContentStatusFromEvent(contents: List[Map[String, AnyRef]]): Map[String, ContentStatus] = { + val enrichedContents = contents.map(content => { + (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) + }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) + .map(x => { + val completedCount = if (x._2.intValue() == 2) 1 else 0 + ContentStatus(x._1, x._2.intValue(), completedCount) + }).groupBy(f => f.contentId) + + enrichedContents.map(content => { + val consumedList = content._2 + val finalStatus = consumedList.map(x => x.status).max + val views = sumFunc(consumedList, (x: ContentStatus) => { + x.viewCount + }) + val completion = sumFunc(consumedList, (x: ContentStatus) => { + x.completedCount + }) + (content._1, ContentStatus(content._1, finalStatus, completion, views)) + }) + } + + /** + * Computation of Sum for viewCount and completedCount. + */ + private def sumFunc(list: List[ContentStatus], valFunc: ContentStatus => Int): Int = list.map(x => valFunc(x)).sum + + + /** + * Method to get the content status from the database + * + * Ex: List(Map("courseId" -> "do_43795", batchId -> "batch1", userId->"user001", contentStatus -> Map("C1"->2, "C2" ->1))) + * + */ + def getContentStatusFromDB(eDataBatch: List[Map[String, AnyRef]], metrics: Metrics): Map[String, UserContentConsumption] = { + + val contentConsumption = scala.collection.mutable.Map[String, UserContentConsumption]() + val primaryFields = Map( + config.userId.toLowerCase() -> eDataBatch.map(x => x(config.userId)).distinct, + config.batchId.toLowerCase -> eDataBatch.map(x => x(config.batchId)).distinct, + config.courseId.toLowerCase -> eDataBatch.map(x => x(config.courseId)).distinct + ) + + val records = Option(readFromDB(primaryFields, config.dbKeyspace, config.dbUserContentConsumptionTable, metrics)) + records.map(record => record.groupBy(col => Map(config.batchId -> col.getObject(config.batchId.toLowerCase()).asInstanceOf[String], config.userId -> col.getObject(config.userId.toLowerCase()).asInstanceOf[String], config.courseId -> col.getObject(config.courseId.toLowerCase()).asInstanceOf[String]))) + .foreach(groupedRecords => groupedRecords.map(entry => { + val identifierMap = entry._1 + val consumptionList = entry._2.flatMap(row => Map(row.getObject(config.contentId.toLowerCase()).asInstanceOf[String] -> Map(config.status -> row.getObject(config.status), config.viewcount -> row.getObject(config.viewcount), config.completedcount -> row.getObject(config.completedcount)))) + .map(entry => { + val contentStatus = entry._2.filter(x => x._2 != null) + val contentId = entry._1 + val status = contentStatus.getOrElse(config.status, 1).asInstanceOf[Number].intValue() + val viewCount = contentStatus.getOrElse(config.viewcount, 0).asInstanceOf[Number].intValue() + val completedCount = contentStatus.getOrElse(config.completedcount, 0).asInstanceOf[Number].intValue() + (contentId, ContentStatus(contentId, status, completedCount, viewCount, false)) + }).toMap + + val userId = identifierMap(config.userId) + val batchId = identifierMap(config.batchId) + val courseId = identifierMap(config.courseId) + + val userContentConsumption = UserContentConsumption(userId, batchId, courseId, consumptionList) + contentConsumption += getUCKey(userContentConsumption) -> userContentConsumption + + })) + contentConsumption.toMap + } + + /** + * Content - AUDIT Event Generation using UserContentConsumption + * "eventsFor" - will have the action (or type) for the event to generate. + */ + def contentAuditEvents(userConsumption: UserContentConsumption): List[TelemetryEvent] = { + val userId = userConsumption.userId + val courseId = userConsumption.courseId + val batchId = userConsumption.batchId + val contentsForEvents = userConsumption.contents.filter(c => c._2.eventsFor.nonEmpty).values + contentsForEvents.flatMap(c => { + c.eventsFor.map(action => { + val properties = if (StringUtils.equalsIgnoreCase(action, config.complete)) Array(config.viewcount, config.completedcount) else Array(config.viewcount) + TelemetryEvent( + actor = ActorObject(id = userId), + edata = EventData(props = properties, `type` = action), // action values are "start", "complete". + context = EventContext(cdata = Array(Map("type" -> config.courseBatch, "id" -> batchId).asJava)), + `object` = EventObject(id = c.contentId, `type` = "Content", rollup = Map[String, String]("l1" -> courseId).asJava) + ) + }) + }).toList + } + + def getDBStatus(collectionId: String): String = { + val requestBody = s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "$collectionId", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + val response = httpUtil.post(config.searchAPIURL, requestBody) + if (response.status == 200) { + val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) + val result = responseBody.getOrDefault("result", new java.util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] + val count = result.getOrDefault("count", 0.asInstanceOf[Number]).asInstanceOf[Number].intValue() + if (count > 0) { + val list = result.getOrDefault("content", new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + list.asScala.head.get("status").asInstanceOf[String] + } else throw new Exception(s"There are no published or retired collection with id: $collectionId") + } else { + logger.error("search-service error: " + response.body) + throw new Exception("search-service not returning error:" + response.status) + } + } + + def getCollectionStatus(collectionId: String): String = { + val cacheStatus = collectionStatusCache.getNonExpired(collectionId).getOrElse("") + if (StringUtils.isEmpty(cacheStatus)) { + val dbStatus = getDBStatus(collectionId) + collectionStatusCache = collectionStatusCache.putClocked(collectionId, dbStatus)._2 + dbStatus + } else cacheStatus + } + + def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (!contentObj.isEmpty) { + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory == "Program" + || primaryCategory == "Curated Program" + || primaryCategory == "Blended Program")) { + isValidProgram = true + } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) + } + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + "/" + courseId + "?fields=identifier,name,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + + def getProgramEvent(eventData: Map[String, AnyRef])( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Map[String, AnyRef] = { + var eventInfoMap: Map[String, AnyRef] = mutable.Map.empty() + var courseId: String = eventData.get(config.courseId).asInstanceOf[String] + var batchId: String = eventData.get(config.batchId).asInstanceOf[String] + var userId: String = eventData.get(config.userId).asInstanceOf[String] + if ((verifyPrimaryCategory(courseId)(metrics, config, httpUtil, cache))) { + eventInfoMap ++= eventData + if (StringUtils.isEmpty(batchId)) { + val row = getEnrolment(userId, courseId)(metrics) + if (row != null) { + eventInfoMap += ("batchId" -> row.getString("batchid")) + } else { + return null; + } + } + eventInfoMap + } else { + null; + } + } + + def getEnrolment(userId: String, courseId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } +} + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala new file mode 100644 index 000000000..4bb6aa2a8 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala @@ -0,0 +1,141 @@ +package org.sunbird.job.programaggregate.functions + +import java.util.UUID + +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.gson.Gson +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.aggregate.common.DeDupHelper +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.{BaseProcessFunction, Metrics} +import org.sunbird.job.programaggregate.domain.{ActorObject, CollectionProgress, EventContext, EventData, EventObject, TelemetryEvent} +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.CassandraUtil + +import scala.collection.JavaConverters._ + +class ProgramProgressCompleteFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]], val stringTypeInfo: TypeInformation[String], @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessFunction[List[CollectionProgress], String](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramProgressCompleteFunction]) + lazy private val gson = new Gson() + var deDupEngine: DeDupEngine = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + cassandraUtil.close() + deDupEngine.close() + super.close() + } + + override def processElement(events: List[CollectionProgress], context: ProcessFunction[List[CollectionProgress], String]#Context, metrics: Metrics): Unit = { + logger.info("events => "+events) + + val pendingEnrolments = if (config.filterCompletedEnrolments) events.filter {p => + val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) + (row != null && row.getInt("status") != 2) + } else events + logger.info("pendingEnrolments =>"+pendingEnrolments) + + val enrolmentQueries = pendingEnrolments.map(enrolmentComplete => getEnrolmentCompleteQuery(enrolmentComplete)) + logger.info("enrolmentQueries => "+enrolmentQueries) + updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + pendingEnrolments.foreach(e => { + createIssueCertEvent(e, context)(metrics) + generateAuditEvent(e, context)(metrics) + }) + logger.info("posting events completed") + // Create and update the checksum to DeDup store for the input events. + if (config.dedupEnabled) { + events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) + .flatten.foreach(checksum => deDupEngine.storeChecksum(checksum)) + } + } + + override def metricsList(): List[String] = { + List(config.dbReadCount, config.dbUpdateCount, config.enrolmentCompleteCount, config.certIssueEventsCount) + } + + def generateAuditEvent(data: CollectionProgress, context: ProcessFunction[List[CollectionProgress], String]#Context)(implicit metrics: Metrics) = { + val auditEvent = TelemetryEvent( + actor = ActorObject(id = data.userId), + edata = EventData(props = Array("status", "completedon"), `type` = "enrol-complete"), // action values are "start", "complete". + context = EventContext(cdata = Array(Map("type" -> config.courseBatch, "id" -> data.batchId).asJava, Map("type" -> "Course", "id" -> data.courseId).asJava)), + `object` = EventObject(id = data.userId, `type` = "User", rollup = Map[String, String]("l1" -> data.courseId).asJava) + ) + logger.info("audit event =>"+gson.toJson(auditEvent)) + context.output(config.auditEventOutputTag, gson.toJson(auditEvent)) + + } + + def getEnrolment(userId: String, courseId: String, batchId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + .and(QueryBuilder.eq("batchid", batchId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getEnrolmentCompleteQuery(enrolment: CollectionProgress): Update.Where = { + logger.info("Enrolment completed for userId: " + enrolment.userId + " batchId: " + enrolment.batchId) + QueryBuilder.update(config.dbKeyspace, config.dbUserEnrolmentsTable) + .`with`(QueryBuilder.set("status", 2)) + .and(QueryBuilder.set("completedon", enrolment.completedOn)) + .and(QueryBuilder.set("progress", enrolment.progress)) + .and(QueryBuilder.set("contentstatus", enrolment.contentStatus.asJava)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + .where(QueryBuilder.eq("userid", enrolment.userId)) + .and(QueryBuilder.eq("courseid", enrolment.courseId)) + .and(QueryBuilder.eq("batchid", enrolment.batchId)) + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + logger.info("is cassandra cluster available =>"+(null !=cassandraUtil.session)) + val result = cassandraUtil.upsert(cqlBatch.toString) + logger.info("result after update => "+result) + if (result) { + metrics.incCounter(config.dbUpdateCount) + metrics.incCounter(config.enrolmentCompleteCount) + } else { + val msg = "Database update has failed" + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } + + /** + * Generation of Certificate Issue event for the enrolment completed users to validate and generate certificate. + * @param enrolment + * @param context + * @param metrics + */ + def createIssueCertEvent(enrolment: CollectionProgress, context: ProcessFunction[List[CollectionProgress], String]#Context)(implicit metrics: Metrics): Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Course Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${enrolment.batchId}_${enrolment.courseId}","type": "CourseCertificateGeneration"},"edata": {"userIds": ["${enrolment.userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${enrolment.batchId}","reIssue": false,"courseId": "${enrolment.courseId}"}}""" + logger.info("o/p event: "+event) + context.output(config.certIssueOutputTag, event) + metrics.incCounter(config.certIssueEventsCount) + } +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala new file mode 100644 index 000000000..fbc2cc45f --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala @@ -0,0 +1,96 @@ +package org.sunbird.job.programaggregate.functions + +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.aggregate.common.DeDupHelper +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.CassandraUtil +import org.sunbird.job.{BaseProcessFunction, Metrics} + +import scala.collection.JavaConverters._ + +class ProgramProgressUpdateFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]], val stringTypeInfo: TypeInformation[String], @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessFunction[List[CollectionProgress], String](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramProgressUpdateFunction]) + var deDupEngine: DeDupEngine = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + cassandraUtil.close() + deDupEngine.close() + super.close() + } + + override def processElement(events: List[CollectionProgress], context: ProcessFunction[List[CollectionProgress], String]#Context, metrics: Metrics): Unit = { + val pendingEnrolments = if (config.filterCompletedEnrolments) events.filter { p => + val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) + (row != null && row.getInt("status") != 2) + } else events + val enrolmentQueries = pendingEnrolments.map(collectionProgress => getEnrolmentUpdateQuery(collectionProgress)) + updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + // Create and update the checksum to DeDup store for the input events. + if (config.dedupEnabled) { + events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) + .flatten.foreach(checksum => deDupEngine.storeChecksum(checksum)) + } + } + + override def metricsList(): List[String] = { + List(config.dbReadCount, config.dbUpdateCount) + } + + def getEnrolment(userId: String, courseId: String, batchId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + .and(QueryBuilder.eq("batchid", batchId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getEnrolmentUpdateQuery(enrolment: CollectionProgress): Update.Where = { + logger.info("Enrolment updated for userId: " + enrolment.userId + " batchId: " + enrolment.batchId) + QueryBuilder.update(config.dbKeyspace, config.dbUserEnrolmentsTable) + .`with`(QueryBuilder.set("status", 1)) + .and(QueryBuilder.set("progress", enrolment.progress)) + .and(QueryBuilder.set("contentstatus", enrolment.contentStatus.asJava)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + .where(QueryBuilder.eq("userid", enrolment.userId)) + .and(QueryBuilder.eq("courseid", enrolment.courseId)) + .and(QueryBuilder.eq("batchid", enrolment.batchId)) + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + val result = cassandraUtil.upsert(cqlBatch.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed" + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala new file mode 100644 index 000000000..6bcb344c2 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -0,0 +1,149 @@ +package org.sunbird.job.programaggregate.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.job.BaseJobConfig +import org.sunbird.job.programaggregate.domain.CollectionProgress + +import java.util + +class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "activity-aggregate-updater") { + + private val serialVersionUID = 2905979434303791379L + + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + implicit val scalaMapTypeInfo: TypeInformation[Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[Map[String, AnyRef]]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]] = TypeExtractor.getForClass(classOf[List[CollectionProgress]]) + + // Kafka Topics Configuration + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaAuditEventTopic: String = config.getString("kafka.output.audit.topic") + val kafkaFailedEventTopic: String = config.getString("kafka.output.failed.topic") + val kafkaCertIssueTopic: String = config.getString("kafka.output.certissue.topic") + + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val activityAggregateUpdaterParallelism: Int = config.getInt("task.activity.agg.parallelism") + val deDupProcessParallelism: Int = config.getInt("task.dedup.parallelism") + val enrolmentCompleteParallelism: Int = config.getInt("task.enrolment.complete.parallelism") + + // Metric List + val totalEventCount = "total-events-count" + val failedEventCount = "failed-events-count" + val dbUpdateCount = "db-update-count" + val dbReadCount = "db-read-count" + val cacheHitCount = "cache-hit-count" + val cacheMissCount = "cache-miss-count" + val batchEnrolmentUpdateEventCount = "batch-enrolment-update-count" + val skipEventsCount = "skipped-events-count" + val processedEnrolmentCount = "processed-enrolment-count" + val enrolmentCompleteCount = "enrolment-complete-count" + val certIssueEventsCount = "cert-issue-events-count" + val retiredCCEventsCount = "retired-consumption-events-count" + + // Cassandra Configurations + val dbUserContentConsumptionTable: String = config.getString("lms-cassandra.consumption.table") + val dbUserActivityAggTable: String = config.getString("lms-cassandra.user_activity_agg.table") + val dbUserEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") + val dbKeyspace: String = config.getString("lms-cassandra.keyspace") + val dbHost: String = config.getString("lms-cassandra.host") + val dbPort: Int = config.getInt("lms-cassandra.port") + + // Redis Configurations + val nodeStore: Int = config.getInt("redis.database.relationCache.id") // Both LeafNodes And Ancestor nodes + val deDupRedisHost: String = config.getString("dedup-redis.host") + val deDupRedisPort: Int = config.getInt("dedup-redis.port") + val deDupStore: Int = config.getInt("dedup-redis.database.index") + val deDupExpirySec: Int = config.getInt("dedup-redis.database.expiry") + + // Tags + val uniqueConsumptionOutputTagName = "unique-consumption-events" + val uniqueConsumptionOutput: OutputTag[Map[String, AnyRef]] = OutputTag[Map[String, AnyRef]](uniqueConsumptionOutputTagName) + val auditEventOutputTagName = "audit-events" + val auditEventOutputTag: OutputTag[String] = OutputTag[String](auditEventOutputTagName) + val failedEventOutputTagName = "failed-events" + val failedEventOutputTag: OutputTag[String] = OutputTag[String](failedEventOutputTagName) + val collectionCompleteOutputTagName = "collection-progress-complete-events" + val collectionCompleteOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionCompleteOutputTagName) + val collectionUpdateOutputTagName = "collection-progress-update-events" + val collectionUpdateOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionUpdateOutputTagName) + val certIssueOutputTagName = "certificate-issue-events" + val certIssueOutputTag: OutputTag[String] = OutputTag[String](certIssueOutputTagName) + + // constants + val activityType = "activity_type" + val activityId = "activity_id" + val contextId = "context_id" + val activityUser = "user_id" + val aggLastUpdated = "agg_last_updated" + val agg = "agg" + val courseId = "courseId" + val batchId = "batchId" + val contentId = "contentId" + val progress = "progress" + val contents = "contents" + val contentStatus = "contentStatus" + val userId = "userId" + val status = "status" + val unitActivityType = "course-unit" + val courseActivityType = "course" + val leafNodes = "leafnodes" + val ancestors = "ancestors" + val viewcount = "viewcount" + val completedcount = "completedcount" + val complete = "complete" + val eData = "edata" + val action = "action" + val batchEnrolmentUpdateCode = "batch-enrolment-update" + val routerFn = "RouterFn" + val consumptionDeDupFn= "consumption-dedup-process" + val programactivityAggregateUpdaterFn = "program-activity-aggregate-updater-fn" + val partition = "partition" + val courseBatch = "CourseBatch" + val collectionProgressUpdateFn = "progress-update-process" + val collectionCompleteFn = "collection-completion-process" + val aggregates = "aggregates" + + // Consumers + val activityAggregateUpdaterConsumer = "activity-aggregate-updater-consumer" + + // Producers + val programactivityAggregateUpdaterProducer = "program-activity-aggregate-updater-audit-events-sink" + val enrolmentCompleteEventProducer = "enrolment-complete-audit-sink" + val programactivityAggFailedEventProducer = "program-activity-aggregate-updater-failed-sink" + val certIssueEventProducer = "certificate-issue-event-producer" + + //Thresholds + val thresholdBatchReadInterval: Int = config.getInt("threshold.batch.read.interval") + val thresholdBatchReadSize: Int = config.getInt("threshold.batch.read.size") + val thresholdBatchWriteSize: Int = config.getInt("threshold.batch.write.size") + val windowShards: Int = config.getInt("task.window.shards") + + + // Job specific configurations + val moduleAggEnabled: Boolean = config.getBoolean("activity.module.aggs.enabled") + val dedupEnabled: Boolean = config.getBoolean("activity.input.dedup.enabled") + val statusCacheExpirySec: Int = config.getInt("activity.collection.status.cache.expiry") + val filterCompletedEnrolments: Boolean = if (config.hasPath("activity.filter.processed.enrolments")) config.getBoolean("activity.filter.processed.enrolments") else true + + // Other services configuration + val searchServiceBasePath: String = config.getString("service.search.basePath") + val searchAPIURL = searchServiceBasePath + "/v3/search" + + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase + "/content/v3/read/" + + val identifier: String = "identifier" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val course: String = "Course" + val parentCollections: String = "parentCollections" + val skippedEventCount = "skipped-event-count" + val defaultHeaders = Map[String, String]("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val name: String = "name" + + +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala new file mode 100644 index 000000000..9c196882a --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -0,0 +1,71 @@ +package org.sunbird.job.programaggregate.task + +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.scala._ +import org.sunbird.job.programaggregate.domain.CollectionProgress +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesFunction} +import org.sunbird.job.util.{FlinkUtil, HttpUtil} + +import java.io.File +import java.util + + +class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregateUpdaterConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]] = TypeExtractor.getForClass(classOf[List[CollectionProgress]]) + + val progressStream = + env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.activityAggregateUpdaterConsumer) + .uid(config.activityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) + .rebalance + .getSideOutput(config.uniqueConsumptionOutput) + .keyBy(new ActivityAggregatorKeySelector(config)) + .countWindow(config.thresholdBatchReadSize) + .process(new ProgramActivityAggregatesFunction(config, httpUtil)) + .name(config.programactivityAggregateUpdaterFn) + .uid(config.programactivityAggregateUpdaterFn) + .setParallelism(config.activityAggregateUpdaterParallelism) + + progressStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) + .name(config.programactivityAggregateUpdaterProducer).uid(config.programactivityAggregateUpdaterProducer) + progressStream.getSideOutput(config.failedEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaFailedEventTopic)) + .name(config.programactivityAggFailedEventProducer).uid(config.programactivityAggFailedEventProducer) + + env.execute(config.jobName) + } + +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object ActivityAggregateUpdaterStreamTask { + + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("program-activity-aggregate-updater.conf").withFallback(ConfigFactory.systemEnvironment())) + val courseAggregator = new ProgramActivityAggregateUpdaterConfig(config) + val kafkaUtil = new FlinkKafkaConnector(courseAggregator) + val httpUtil = new HttpUtil + val task = new ProgramActivityAggregateUpdaterStreamTask(courseAggregator, kafkaUtil, httpUtil) + task.process() + } + +} +// $COVERAGE-ON$ + +class ActivityAggregatorKeySelector(config: ProgramActivityAggregateUpdaterConfig) extends KeySelector[Map[String, AnyRef], Int] { + private val serialVersionUID = 7267989625042068736L + private val shards = config.windowShards + override def getKey(in: Map[String, AnyRef]): Int = { + in.getOrElse(config.userId, "").asInstanceOf[String].hashCode % shards + } +} diff --git a/program-activity-aggregate-updater/src/test/resources/logback-test.xml b/program-activity-aggregate-updater/src/test/resources/logback-test.xml new file mode 100644 index 000000000..2e5cb5e09 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/logback-test.xml @@ -0,0 +1,18 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/program-activity-aggregate-updater/src/test/resources/test.conf b/program-activity-aggregate-updater/src/test/resources/test.conf new file mode 100644 index 000000000..f036ad5e2 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/test.conf @@ -0,0 +1,53 @@ +include "base-test.conf" + +kafka { + input.topic = "sunbirddev.coursebatch.job.request" + output.audit.topic = "sunbirddev.telemetry.raw" + output.failed.topic = "sunbirddev.activity.agg.failed" + output.certissue.topic = "sunbirddev.issue.certificate.request" + groupId = "sunbirddev-activity-aggregate-updater-group" +} + +task { + window.shards = 1 + consumer.parallelism = 1 + dedup.parallelism = 1 + activity.agg.parallelism = 1 + enrolment.complete.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + consumption.table = "user_content_consumption" + user_activity_agg.table = "user_activity_agg" + user_enrolments.table = "user_enrolments" +} + +redis { + database { + relationCache.id = 10 + } +} + +threshold.batch.read.interval = 60 // In sec +threshold.batch.read.size = 1 +threshold.batch.write.size = 5 + +dedup-redis { + host = localhost + port = 6340 + database.index = 13 + database.expiry = 600 +} + +activity { + module.aggs.enabled = true + input.dedup.enabled = true + collection.status.cache.expiry = 3600 +} + +service { + search { + basePath = "http://search-service:9000" + } +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/test/resources/test.cql b/program-activity-aggregate-updater/src/test/resources/test.cql new file mode 100644 index 000000000..5fff30d90 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/test.cql @@ -0,0 +1,79 @@ +CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE sunbird_courses.user_content_consumption ( + userid text, + courseid text, + batchid text, + contentid text, + completedcount int, + datetime timestamp, + lastaccesstime text, + lastcompletedtime text, + lastupdatedtime text, + progress int, + status int, + viewcount int, + PRIMARY KEY (userid, courseid, batchid, contentid) +) WITH CLUSTERING ORDER BY (courseid ASC, batchid ASC, contentid ASC); + +// EVENT_1 Testcase data +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012299','0126083288437637121','do_1127212344324751361295',100, 2, 3,1) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012300','0126083288437637121','do_1127212344324751361295',100, 2, 2,2) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012301','0126083288437637121','do_1127212344324751361295',0, 1,0,1) ; + +//Event_2 Testcase Data +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R1','Batch1','course001',100, 2,1,0) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R2','Batch1','course001',100, 2,1,0) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R3','Batch1','course001',100, 2,1,0) ; + + + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg ( + activity_id text, + user_id text, + activity_type text, + context_id text, + agg Map, + aggregates Map, + agg_last_updated Map, + PRIMARY KEY ((activity_type, activity_id), context_id, user_id) +); + +CREATE TABLE sunbird_courses.user_enrolments ( + userid text, + courseid text, + batchid text, + active boolean, + addedby text, + certificates list>>, + completedon timestamp, + completionpercentage int, + contentstatus map, + datetime timestamp, + enrolleddate text, + issued_certificates list>>, + lastreadcontentid text, + lastreadcontentstatus int, + progress int, + status int, + PRIMARY KEY (userid, courseid, batchid) +) WITH CLUSTERING ORDER BY (courseid ASC, batchid ASC) + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} + AND comment = '' + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} + AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.1 + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair_chance = 0.0 + AND speculative_retry = '99PERCENTILE'; +CREATE INDEX inx_ues_status ON sunbird_courses.user_enrolments (status); +CREATE INDEX inx_ues_certs ON sunbird_courses.user_enrolments (values(certificates)); + + +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('user001', 'course001', 'Batch1', 1) diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala new file mode 100644 index 000000000..a26f3426b --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala @@ -0,0 +1,194 @@ +package org.sunbird.job.fixture + +object EventFixture { + + /** + * case-1. Inserting a first course which is having 3 leaf nodes in the redis database and cassandra database + * does not contains this course id. + * courseId = + * ============BE_JOB_REQUEST_CONTENTS========== + * do_1127212344324751361295 - course + * do_course_unit1 - unit1 + * do_11260735471149056012299 - resource + * do_course_unit2 - unit2 + * do_11260735471149056012300 - resource + * do_course_unit3 - unit3 + * do_11260735471149056012301 - resource + * do_11260735471149056012300 -resource + * + * + * ============== content status in the event ====== + * do_11260735471149056012299 - 2 + * do_11260735471149056012301 - 1 + * do_11260735471149056012300 - 1 + * ============== content status in the database(content-consumption) + * do_11260735471149056012299 - 2 + * do_11260735471149056012301 - 1 + * do_11260735471149056012300 - 2 + * + * ============== Computation ============== + * unit level computation + * do_11260735471149056012299:ansestor -> do_course_unit1, do_1127212344324751361295 + * do_course_unit1:do_1127212344324751361295:leafnodes: do_11260735471149056012299 + * lefNodesSize = 1, completed = 1 + * 1/1 = 100% + * + * do_11260735471149056012301:ansestor -> do_course_unit3,do_1127212344324751361295 + * do_course_unit3:do_1127212344324751361295:leafNodes -> do_11260735471149056012301,do_11260735471149056012300 + * leafNodesSize = 2,completed 1 + * 1/2 = 50% + * + * do_11260735471149056012300:ansestor -> do_course_unit3,do_course_unit2,do_1127212344324751361295 + * do_course_unit3:do_1127212344324751361295:leafNodes -> do_11260735471149056012301,do_11260735471149056012300 + * leafNodesSize = 2, completed = 1 + * 1/2 = 50% + * + * do_course_unit2:do_1127212344324751361295:leafNodes -> do_11260735471149056012300 + * leafNodesSize = 1, completed = 1 + * 1/1 = 100% + * + * course level progress computation + * do_1127212344324751361295:leafNodes = do_11260735471149056012299, do_11260735471149056012301, do_11260735471149056012300 + */ + + val EVENT_1: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-enrolment-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val courseLeafNodes = Map("do_1127212344324751361295:do_1127212344324751361295:leafnodes" -> List("do_11260735471149056012299", "do_11260735471149056012300", "do_11260735471149056012301")) + val unitLeafNodes_1 = Map("do_1127212344324751361295:do_course_unit1:leafnodes" -> List("do_11260735471149056012299")) + val unitLeafNodes_2 = Map("do_1127212344324751361295:do_course_unit2:leafnodes" -> List("do_11260735471149056012300")) + val unitLeafNodes_3 = Map("do_1127212344324751361295:do_course_unit3:leafnodes" -> List("do_11260735471149056012301", "do_11260735471149056012300")) + + val ancestorsResource_1 = Map("do_1127212344324751361295:do_11260735471149056012299:ancestors" -> List("do_course_unit1", "do_1127212344324751361295")) + val ancestorsResource_2 = Map("do_1127212344324751361295:do_11260735471149056012300:ancestors" -> List("do_course_unit2", "do_course_unit3", "do_1127212344324751361295")) + val ancestorsResource_3 = Map("do_1127212344324751361295:do_11260735471149056012301:ancestors" -> List("do_course_unit3", "do_1127212344324751361295")) + + val CASE_1:Map[String, AnyRef] = Map("event" -> EVENT_1, "cacheData" -> List(courseLeafNodes, + unitLeafNodes_1, unitLeafNodes_2,unitLeafNodes_3, ancestorsResource_1,ancestorsResource_2,ancestorsResource_3)) + + + + val EVENT_2: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R1","status":2},{"contentId":"do_R2","status":1},{"contentId":"do_R3","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + + /** **** course structure **** + * + * case2: When all resource progress is 2 in the content-consumption table + * + * course001 - course + * unit1 - unit1 + * do_R1 - Resource + * do_R3 - Resource + * unit2 - Unit2 + * do_R2 - Resource + * do_R3 - Resource + * + * ============== content status in the event ====== + * do_R1 - 1 + * do_R3 - 1 + * do_R2 - 1 + * ============== content status in the database(content-consumption) + * do_R1 - 2 + * do_R2 - 2 + * do_R3 - 2 + * + * //Unit Level + * course001:do_R1:ansestor => unit1,course001 + * unit1:leafNodes -> do_R1,do_R3 + * output:leafNodesSize = 2, completed=2 + * course001:do_R2:ansestor => unit2,course001 + * unit2:leafNodes -> do_R2,do_R3 + * output:leafNodesSize = 2, completed = 2 + * course001:do_R3:ansestor => unit1, unit2,course001 + * unit1:leafNodes -> do_R1,do_R3 + * unit2:leafNodes -> do_R2,do_R3 + * output:leafNodes=2, completed=2 + * // CourseLevel + * output:LeafNodes =3, Completed =3 + * + * + */ + val e2_courseLeafNodes = Map("course001:course001:leafnodes" -> List("do_R1", "do_R3", "do_R2")) + val e2_unitLeafNodes_1 = Map("course001:unit1:leafnodes" -> List("do_R1", "do_R3")) + val e2_unitLeafNodes_2 = Map("course001:unit2:leafnodes" -> List("do_R2", "do_R3")) + + val e2_ancestorsResource_1 = Map("course001:do_R1:ancestors" -> List("unit1", "course001")) + val e2_ancestorsResource_2 = Map("course001:do_R3:ancestors" -> List("unit1", "unit2", "course001")) + val e2_ancestorsResource_3 = Map("course001:do_R2:ancestors" -> List("unit2", "course001")) + + val CASE_2:Map[String, AnyRef] = Map("event" -> EVENT_2, "cacheData" -> List(e2_courseLeafNodes, + e2_unitLeafNodes_1, e2_unitLeafNodes_2,e2_ancestorsResource_1, e2_ancestorsResource_2,e2_ancestorsResource_3)) + + /** * + * + * Case3: When resource data is not available in the content_consumption table and user_activity_agg + * + * C11 - course + * unit11 - unit1 + * R11 - Resource + * R22 - Resource + * unit22 - Unit2 + * R11 - Resource + * + * ============== content status in the event ====== + * R11 - 2 + * R22 - 2 + * ============== content status in the database(content-consumption) + * Data is not available + * + * //Unit Level + * C11:R11:ansestor => unit11,unit22,C11 + * unit11:leafNodes -> R11,R22 + * output:leafNodesSize = 2, completed=2 + * unit11:leafNodes -> R11 + * output:leafNodesSize = 1, completed=1 + * C11:R22:ansestor => unit11,C11 + * unit11:leafNodes -> R11,R22 + * output:leafNodesSize = 2, completed = 2 + * // CourseLevel + * output:LeafNodes =2, Completed =2 + * + */ + + val e3_courseLeafNodes = Map("C11:C11:leafnodes" -> List("R11", "R22")) + val e3_unitLeafNodes_1 = Map("C11:unit11:leafnodes" -> List("R11", "R22")) + val e3_unitLeafNodes_2 = Map("C11:unit22:leafnodes" -> List("R11")) + + val e3_ancestorsResource_1 = Map("C11:R11:ancestors" -> List("unit11", "C11")) + val e3_ancestorsResource_2 = Map("C11:R11:ancestors" -> List("unit22", "C11")) + val e3_ancestorsResource_3 = Map("C11:R22:ancestors" -> List("unit11", "C11")) + + val EVENT_3: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"R11","status":2},{"contentId":"R22","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"B11","userId":"U11","courseId":"C11"}} + |""".stripMargin + + val CASE_3:Map[String, AnyRef] = Map("event" -> EVENT_3, "cacheData" -> List(e3_courseLeafNodes, e3_unitLeafNodes_1, e3_unitLeafNodes_2,e3_ancestorsResource_1, e3_ancestorsResource_2, e3_ancestorsResource_3) ) + + val EVENT_4: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-enrolment-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val EVENT_5: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val CC_EVENT1: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac81","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R1","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + val CC_EVENT2: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac82","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R2","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + val CC_EVENT3: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac83","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R3","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala new file mode 100644 index 000000000..cd881c3ab --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala @@ -0,0 +1,59 @@ +package org.sunbird.job.spec + +import java.util + +import org.apache.flink.streaming.api.functions.sink.SinkFunction + + +class AuditEventSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + AuditEventSink.values.add(value) + } + } +} + +object AuditEventSink { + val values: util.List[String] = new util.ArrayList() +} + +class FailedEventSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + FailedEventSink.values.add(value) + } + } +} + +object FailedEventSink { + val values: util.List[String] = new util.ArrayList() +} + +class SuccessEvent extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + SuccessEventSink.values.add(value) + } + } +} + +object SuccessEventSink { + val values: util.List[String] = new util.ArrayList() +} + + +class CertificateIssuedEventsSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + CertificateIssuedEvents.values.add(value) + } + } +} + +object CertificateIssuedEvents { + val values: util.List[String] = new util.ArrayList() +} diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala new file mode 100644 index 000000000..2194ab23d --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala @@ -0,0 +1,219 @@ +package org.sunbird.job.spec + +import java.util +import com.datastax.driver.core.Row +import com.google.gson.Gson +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext +import org.apache.flink.test.util.MiniClusterWithClientResource +import org.cassandraunit.CQLDataLoader +import org.cassandraunit.dataset.cql.FileCQLDataSet +import org.cassandraunit.utils.EmbeddedCassandraServerHelper +import org.mockito.Mockito +import org.mockito.Mockito._ +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.fixture.EventFixture +import org.sunbird.job.programaggregate.task.{ProgramActivityAggregateUpdaterConfig, ProgramActivityAggregateUpdaterStreamTask} +import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil} +import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} +import redis.clients.jedis.Jedis +import redis.embedded.RedisServer + +import scala.collection.mutable +import scala.collection.JavaConverters._ + +class ProgramActivityAggregateUpdaterTaskTestSpec extends BaseTestSpec { + + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + + var redisServer: RedisServer = _ + redisServer = new RedisServer(6340) + redisServer.start() + var jedis: Jedis = _ + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + val gson = new Gson() + val config: Config = ConfigFactory.load("test.conf") + val courseAggregatorConfig: ProgramActivityAggregateUpdaterConfig = new ProgramActivityAggregateUpdaterConfig(config) + val mockHttpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) + + var cassandraUtil: CassandraUtil = _ + + val requestBody = s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "course001", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + override protected def beforeAll(): Unit = { + super.beforeAll() + val redisConnect = new RedisConnect(courseAggregatorConfig) + jedis = redisConnect.getConnection(courseAggregatorConfig.nodeStore) + EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) + cassandraUtil = new CassandraUtil(courseAggregatorConfig.dbHost, courseAggregatorConfig.dbPort) + val session = cassandraUtil.session + + val dataLoader = new CQLDataLoader(session) + dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) + // Clear the metrics + testCassandraUtil(cassandraUtil) + BaseMetricsReporter.gaugeMetrics.clear() + jedis.flushDB() + flinkCluster.before() + updateRedis(jedis, EventFixture.CASE_1.asInstanceOf[Map[String, AnyRef]]) + updateRedis(jedis, EventFixture.CASE_2.asInstanceOf[Map[String, AnyRef]]) + updateRedis(jedis, EventFixture.CASE_3.asInstanceOf[Map[String, AnyRef]]) + } + + override protected def afterAll(): Unit = { + super.afterAll() + try { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() + redisServer.stop() + } catch { + case ex: Exception => { + } + } + flinkCluster.after() + } + + def initialize() { + when(mockKafkaUtil.kafkaMapSource(courseAggregatorConfig.kafkaInputTopic)).thenReturn(new CompleteContentConsumptionMapSource) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaAuditEventTopic)).thenReturn(new AuditEventSink) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaFailedEventTopic)).thenReturn(new FailedEventSink) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaCertIssueTopic)).thenReturn(new CertificateIssuedEventsSink) + } + + "Activity Aggregator " should " compute and update enrolment as completed when all the content consumption data processed" in { + initialize() + new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, new HttpUtil).process() + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.totalEventCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.batchEnrolmentUpdateEventCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.dbReadCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.dbUpdateCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.cacheHitCount}").getValue() should be(18) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.processedEnrolmentCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.enrolmentCompleteCount}").getValue() should be(1) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.failedEventCount}").getValue() should be(0) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.skipEventsCount}").getValue() should be(0) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.cacheMissCount}").getValue() should be(0) + + AuditEventSink.values.size() should be(4) + AuditEventSink.values.forEach(event => { + println("AUDIT_TELEMETRY_EVENT: " + event) + }) + jedis.select(courseAggregatorConfig.deDupStore) + val deDupKeys = jedis.keys("*") + println("DeDup Keys:" + deDupKeys) + deDupKeys.size() should be (3) + jedis.select(courseAggregatorConfig.nodeStore) + } + + "Activity Aggregator " should " throw exception when the cache not available for root collection" in { + jedis.select(courseAggregatorConfig.nodeStore) + jedis.flushAll() + when(mockHttpUtil.post(courseAggregatorConfig.searchAPIURL, requestBody)).thenReturn(HTTPResponse(200, """{"id":"api.v1.search","ver":"1.0","ts":"2020-12-16T12:37:40.283Z","params":{"resmsgid":"7c4cf0b0-3f9b-11eb-9b0c-abcfbdf41bc3","msgid":"7c4b1bf0-3f9b-11eb-9b0c-abcfbdf41bc3","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"count":1,"content":[{"identifier":"course001","objectType":"Content","status":"Live"}]}}""")) + initialize() + + val activityAggTask = new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, mockHttpUtil) + the [Exception] thrownBy { + activityAggTask.process() + } should have message "Job execution failed." + + // De-dup should not save the keys for which the processing failed. + // This will help in processing the same data after restart. + jedis.select(courseAggregatorConfig.deDupStore) + jedis.keys("*").size() should be (0) + jedis.select(courseAggregatorConfig.nodeStore) + + FailedEventSink.values.forEach(event => { + println("FAILED_EVENT_DATA: " + event) + }) + // failedEventSink.values.size() should be (2) + } + + ignore should " skip the retired collection consumption events" in { + jedis.select(courseAggregatorConfig.nodeStore) + jedis.flushAll() + reset(mockHttpUtil) + when(mockHttpUtil.post(courseAggregatorConfig.searchAPIURL, requestBody)).thenReturn(HTTPResponse(200, """{"id":"api.v1.search","ver":"1.0","ts":"2020-12-16T12:37:40.283Z","params":{"resmsgid":"7c4cf0b0-3f9b-11eb-9b0c-abcfbdf41bc3","msgid":"7c4b1bf0-3f9b-11eb-9b0c-abcfbdf41bc3","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"count":1,"content":[{"identifier":"course001","objectType":"Content","status":"Retired"}]}}""")) + initialize() + + val activityAggTask = new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, mockHttpUtil) + activityAggTask.process() + + jedis.select(courseAggregatorConfig.deDupStore) + jedis.keys("*").size() should be (0) + jedis.select(courseAggregatorConfig.nodeStore) + + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.retiredCCEventsCount}").getValue() should be(3) + + } + + def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { + cassandraUtil.reconnect() + } + + def updateRedis(jedis: Jedis, testData: Map[String, AnyRef]) { + testData.get("cacheData").map(data => { + data.asInstanceOf[List[Map[String, AnyRef]]].map(cacheData => { + cacheData.map(x => { + x._2.asInstanceOf[List[String]].foreach(d => { + jedis.sadd(x._1, d) + }) + }) + }) + }) + } + + def readFromCassandra(event: String): util.List[Row] = { + val event1_primaryCols = getPrimaryCols(gson.fromJson(event, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala.asJava) + val query = s"select * from sunbird_courses.user_activity_agg where context_id='cb:${event1_primaryCols.get("batchid").get}' and user_id='${event1_primaryCols.get("userid").get}' ALLOW FILTERING;" + cassandraUtil.find(query) + } + + def readFromContentConsumptionTable(event: String): util.List[Row] = { + val event1_primaryCols = getPrimaryCols(gson.fromJson(event, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala.asJava) + val query = s"select * from sunbird_courses.user_content_consumption where userid='${event1_primaryCols.get("userid").get}' and batchid='${event1_primaryCols.get("batchid").get}' and courseid='${event1_primaryCols.get("courseid").get}' ALLOW FILTERING;" + cassandraUtil.find(query) + } + + + def getPrimaryCols(event: util.Map[String, AnyRef]): mutable.Map[String, String] = { + val eventData = event.get("edata").asInstanceOf[util.Map[String, AnyRef]] + val primaryFields = List("userid", "courseid", "batchid") + eventData.asScala.map(v => (v._1.toLowerCase, v._2)).filter(x => primaryFields.contains(x._1)).asInstanceOf[mutable.Map[String, String]] + } +} + +private class CompleteContentConsumptionMapSource extends SourceFunction[util.Map[String, AnyRef]] { + + override def run(ctx: SourceContext[util.Map[String, AnyRef]]) { + ctx.collect(jsonToMap(EventFixture.CC_EVENT1)) + ctx.collect(jsonToMap(EventFixture.CC_EVENT2)) + ctx.collect(jsonToMap(EventFixture.CC_EVENT3)) + } + + override def cancel() = {} + + def jsonToMap(json: String): util.Map[String, AnyRef] = { + val gson = new Gson() + gson.fromJson(json, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]] + } + +} From 97c24d6600c5c413c3a3dba3f95de837a1b7f9b6 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 01:01:51 +0530 Subject: [PATCH 35/88] Type cast issue fix --- .../functions/IssueCertificateHelper.scala | 10 ++++++++-- .../functions/IssueCertificateHelper.scala | 11 ----------- .../helpers/PostPublishRelationUpdater.scala | 5 +---- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 794e43da5..70428342f 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -216,6 +216,12 @@ trait IssueCertificateHelper { val dateFormatter = new SimpleDateFormat("yyyy-MM-dd") val related = getRelatedData(event, enrolledUser, assessedUser, userDetails, additionalProps, certName, courseName)(config) val providerName = getCourseOrganisation(event.courseId)(metrics, config, cache, httpUtil) + val parentCollections: List[String] = Option(courseInfo.get(config.parentCollections)) + .collect { + case list: java.util.List[_] => + list.asInstanceOf[java.util.List[String]].asScala.toList + } + .getOrElse(List.empty) val eData = Map[String, AnyRef] ( "issuedDate" -> dateFormatter.format(enrolledUser.issuedOn), "data" -> List(Map[String, AnyRef]("recipientName" -> recipientName, "recipientId" -> event.userId)), @@ -234,7 +240,7 @@ trait IssueCertificateHelper { "providerName" -> providerName, "tag" -> event.batchId, "primaryCategory" -> courseInfo.getOrElse("primaryCategory", "").asInstanceOf[String], - "parentCollections" -> courseInfo.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]], + "parentCollections" -> parentCollections, "coursePosterImage" -> courseInfo.getOrElse("coursePosterImage", "").asInstanceOf[String], ) @@ -266,7 +272,7 @@ trait IssueCertificateHelper { val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') - val parentCollections = response.getOrElse("parentCollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] + val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] Map( "courseId" -> courseId, "courseName" -> courseName, diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala index d23482e8b..3220838bf 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala @@ -30,15 +30,4 @@ trait IssueCertificateHelper { throw new Exception(s"Error from get API : ${url}, with response: ${response}") } } - - def getCourseReadDetails(courseId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.ArrayList[String] = { - val courseMetadata = cache.getWithRetry(courseId) - if (null == courseMetadata || courseMetadata.isEmpty) { - val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=primaryCategory,identifier,parentCollections" - val response = getAPICall(url, "content")(config, httpUtil, metrics) - response.getOrElse(config.parentCollections, new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] - } else { - courseMetadata.getOrElse(config.parentCollections.toLowerCase(), new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] - } - } } diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala index 7badd99fc..8165e3c3e 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -73,8 +73,7 @@ trait PostPublishRelationUpdater { ) .filter(_ >= ' ') val parentCollections = response - .getOrElse("parentCollections", new java.util.ArrayList()) - .asInstanceOf[java.util.ArrayList[String]] + .getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) @@ -84,11 +83,9 @@ trait PostPublishRelationUpdater { courseInfoMap.put(config.versionKey, versionKey) courseInfoMap } else { - logger.info("Content read response from cache " + JSONUtil.serialize(courseMetadata)) val name = courseMetadata.getOrElse(config.name, "").asInstanceOf[String] val category = courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String] val version = courseMetadata.getOrElse("versionkey", "").asInstanceOf[String] - logger.info("name: " + name + ", category: " + category + ", version: " + version) val parentCollections = courseMetadata .getOrElse("parentcollections", new java.util.ArrayList()) .asInstanceOf[java.util.ArrayList[String]] From 68bede05eb778fe205303fc111821b62ad302de2 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 03:20:59 +0530 Subject: [PATCH 36/88] Added logs --- .../job/collectioncert/functions/IssueCertificateHelper.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 70428342f..dec340dde 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -222,6 +222,7 @@ trait IssueCertificateHelper { list.asInstanceOf[java.util.List[String]].asScala.toList } .getOrElse(List.empty) + logger.info("Received parentCollections from getCourseInfo : " + JSONUtil.serialize(parentCollections)) val eData = Map[String, AnyRef] ( "issuedDate" -> dateFormatter.format(enrolledUser.issuedOn), "data" -> List(Map[String, AnyRef]("recipientName" -> recipientName, "recipientId" -> event.userId)), @@ -269,6 +270,7 @@ trait IssueCertificateHelper { val courseInfoMap = if (null == courseMetadata || courseMetadata.isEmpty) { val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory,posterImage" val response = getAPICall(url, "content")(config, httpUtil, metrics) + logger.info("content is read from API. response : " + JSONUtil.serialize(response)) val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') @@ -281,6 +283,7 @@ trait IssueCertificateHelper { "coursePosterImage" -> posterImage, ) } else { + logger.info("content is read from Cache. response : " + JSONUtil.serialize(courseMetadata)) val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') val parentCollections = courseMetadata.getOrElse("parentcollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] From 5fb91d5621d83f20f092a1e98750eb2362e3fa83 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 04:27:50 +0530 Subject: [PATCH 37/88] Added the imports --- .../job/collectioncert/functions/IssueCertificateHelper.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index dec340dde..7d55bf2f2 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -11,7 +11,7 @@ import org.sunbird.job.Metrics import org.sunbird.job.cache.DataCache import org.sunbird.job.collectioncert.domain.{AssessedUser, AssessmentUserAttempt, BEJobRequestEvent, EnrolledUser, Event, EventObject} import org.sunbird.job.collectioncert.task.CollectionCertPreProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil, JSONUtil} import scala.collection.JavaConverters._ From e1d627631aa722f3afd6f15613fe8cbff8b9cd25 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 06:56:43 +0530 Subject: [PATCH 38/88] Fix for reading data from getCourseInfo --- .../functions/IssueCertificateHelper.scala | 52 +++++++++++-------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 7d55bf2f2..7004d742e 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -211,11 +211,10 @@ trait IssueCertificateHelper { val lastName = Option(userDetails.getOrElse("lastName", "").asInstanceOf[String]).getOrElse("") def nullStringCheck(name:String):String = {if(StringUtils.equalsIgnoreCase("null", name)) "" else name} val recipientName = nullStringCheck(firstName).concat(" ").concat(nullStringCheck(lastName)).trim - val courseInfo: Map[String, Any] = getCourseInfo(event.courseId)(metrics, config, cache, httpUtil) - val courseName = courseInfo.getOrElse("courseName", "").asInstanceOf[String] + val courseInfo: java.util.Map[String, AnyRef] = getCourseInfo(event.courseId)(metrics, config, cache, httpUtil) + val courseName = courseInfo.getOrDefault("courseName", "").asInstanceOf[String] val dateFormatter = new SimpleDateFormat("yyyy-MM-dd") val related = getRelatedData(event, enrolledUser, assessedUser, userDetails, additionalProps, certName, courseName)(config) - val providerName = getCourseOrganisation(event.courseId)(metrics, config, cache, httpUtil) val parentCollections: List[String] = Option(courseInfo.get(config.parentCollections)) .collect { case list: java.util.List[_] => @@ -238,11 +237,11 @@ trait IssueCertificateHelper { "basePath" -> config.certBasePath, "related" -> related, "name" -> certName, - "providerName" -> providerName, + "providerName" -> courseInfo.getOrDefault("providerName", "").asInstanceOf[String], "tag" -> event.batchId, - "primaryCategory" -> courseInfo.getOrElse("primaryCategory", "").asInstanceOf[String], + "primaryCategory" -> courseInfo.getOrDefault("primaryCategory", "").asInstanceOf[String], "parentCollections" -> parentCollections, - "coursePosterImage" -> courseInfo.getOrElse("coursePosterImage", "").asInstanceOf[String], + "coursePosterImage" -> courseInfo.getOrDefault("coursePosterImage", "").asInstanceOf[String], ) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) @@ -265,9 +264,9 @@ trait IssueCertificateHelper { locationProps ++ enrolledUser.additionalProps ++ assessedUser.additionalProps ++ userAdditionalProps ++ courseAdditionalProps } - def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): Map[String, Any] = { + def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { val courseMetadata = cache.getWithRetry(courseId) - val courseInfoMap = if (null == courseMetadata || courseMetadata.isEmpty) { + if (null == courseMetadata || courseMetadata.isEmpty) { val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory,posterImage" val response = getAPICall(url, "content")(config, httpUtil, metrics) logger.info("content is read from API. response : " + JSONUtil.serialize(response)) @@ -275,27 +274,34 @@ trait IssueCertificateHelper { val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] - Map( - "courseId" -> courseId, - "courseName" -> courseName, - "parentCollections" -> parentCollections, - "primaryCategory" -> primaryCategory, - "coursePosterImage" -> posterImage, - ) + val orgData = response.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length-1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap } else { logger.info("content is read from Cache. response : " + JSONUtil.serialize(courseMetadata)) val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') val parentCollections = courseMetadata.getOrElse("parentcollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse("posterimage", "").asInstanceOf[String]).filter(_ >= ' ') - Map( - "courseId" -> courseId, - "courseName" -> courseName, - "parentCollections" -> parentCollections, - "primaryCategory" -> primaryCategory, - "coursePosterImage" -> posterImage, - ) + val orgData = courseMetadata.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length-1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap } - courseInfoMap } } From 441f4a28950a4f90ecf62b052fdc1293818bec4a Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 10:39:58 +0530 Subject: [PATCH 39/88] Moved generating prog cert event before generating notification --- .../functions/IssueCertificateHelper.scala | 4 ++-- .../functions/CertificateGeneratorFunction.scala | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 7004d742e..dcd23f57a 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -221,7 +221,7 @@ trait IssueCertificateHelper { list.asInstanceOf[java.util.List[String]].asScala.toList } .getOrElse(List.empty) - logger.info("Received parentCollections from getCourseInfo : " + JSONUtil.serialize(parentCollections)) + val eData = Map[String, AnyRef] ( "issuedDate" -> dateFormatter.format(enrolledUser.issuedOn), "data" -> List(Map[String, AnyRef]("recipientName" -> recipientName, "recipientId" -> event.userId)), @@ -243,7 +243,7 @@ trait IssueCertificateHelper { "parentCollections" -> parentCollections, "coursePosterImage" -> courseInfo.getOrDefault("coursePosterImage", "").asInstanceOf[String], ) - + logger.info("Constructured eData from preProcessor : " + JSONUtil.serialize(eData)) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index bc69b8581..45367b75e 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -78,13 +78,7 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: metrics.incCounter(config.skippedEventCount) logger.info(s"Certificate already issued for: ${event.eData.getOrElse("userId", "")} ${event.related}") } - logger.info("Certificate Issued for '"+ event.primaryCategory + "', parentCollections value ? " + event.parentCollections) - if ("Course".equalsIgnoreCase(event.primaryCategory) && !event.parentCollections.isEmpty) { - val courseCompletionEvent = generateCourseCompletionEvent(event) - Option(courseCompletionEvent ).map(e => { - context.output(config.generateProgramCertificateOutputTag, courseCompletionEvent) - }) - } + logger.info("Certificate Issued for '"+ event.primaryCategory + "', parentCollections value ? " + event.parentCollections) metrics.incCounter(config.successEventCount) } catch { case e: Exception => @@ -316,6 +310,12 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val audit = ScalaJsonUtil.serialize(certificateAuditEvent) context.output(config.auditEventOutputTag, audit) logger.info("pushAuditEvent: certificate audit event success {}", audit) + if ("Course".equalsIgnoreCase(event.primaryCategory) && !event.parentCollections.isEmpty) { + val courseCompletionEvent = generateCourseCompletionEvent(event) + Option(courseCompletionEvent ).map(e => { + context.output(config.generateProgramCertificateOutputTag, courseCompletionEvent) + }) + } context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { From 3bfced4ffc73da8a021ff9d4ffb1005f1d0c2ad8 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Wed, 4 Oct 2023 12:21:53 +0530 Subject: [PATCH 40/88] Added logs --- .../functions/CertificateGeneratorFunction.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index 45367b75e..c8baafdf5 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -19,7 +19,7 @@ import org.sunbird.job.certgen.domain._ import org.sunbird.job.certgen.exceptions.ServerException import org.sunbird.job.certgen.task.CertificateGeneratorConfig import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil, ScalaJsonUtil, JSONUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} import java.io.{File, IOException} @@ -73,12 +73,10 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: if(certValidator.isNotIssued(event)(config, metrics, cassandraUtil)) { if(config.enableRcCertificate) generateCertificateUsingRC(event, context)(metrics) else generateCertificate(event, context)(metrics) - } else { metrics.incCounter(config.skippedEventCount) logger.info(s"Certificate already issued for: ${event.eData.getOrElse("userId", "")} ${event.related}") } - logger.info("Certificate Issued for '"+ event.primaryCategory + "', parentCollections value ? " + event.parentCollections) metrics.incCounter(config.successEventCount) } catch { case e: Exception => @@ -313,8 +311,14 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: if ("Course".equalsIgnoreCase(event.primaryCategory) && !event.parentCollections.isEmpty) { val courseCompletionEvent = generateCourseCompletionEvent(event) Option(courseCompletionEvent ).map(e => { + logger.info("generated course completion event : " + JSONUtil.serialize(courseCompletionEvent)) context.output(config.generateProgramCertificateOutputTag, courseCompletionEvent) + }).getOrElse({ + logger.info("Failed to generate program cert. Option is empty.") }) + logger.info("Certificate Issued for '"+ event.primaryCategory + "', parentCollections value ? " + event.parentCollections) + } else { + logger.info("Skipping event to generate program certificate.") } context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) From aaccd3232a24f229bdd86f2bef4382aa202096ad Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 4 Oct 2023 12:49:10 +0530 Subject: [PATCH 41/88] Added the loggers for IssueCertificateHelper.scala --- .../collectioncert/functions/IssueCertificateHelper.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index dcd23f57a..2abf4eba7 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -24,13 +24,16 @@ trait IssueCertificateHelper { val criteria = validateTemplate(template, event.batchId)(config) //validateEnrolmentCriteria val certName = template.getOrElse(config.name, "") + logger.info("CertName" + certName) val additionalProps: Map[String, List[String]] = ScalaJsonUtil.deserialize[Map[String, List[String]]](template.getOrElse("additionalProps", "{}")) val enrolledUser: EnrolledUser = validateEnrolmentCriteria(event, criteria.getOrElse(config.enrollment, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], certName, additionalProps)(metrics, cassandraUtil, config) + logger.info("enrolledUser" + enrolledUser) //validateAssessmentCriteria val assessedUser = validateAssessmentCriteria(event, criteria.getOrElse(config.assessment, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], enrolledUser.userId, additionalProps)(metrics, cassandraUtil, contentCache, config) + logger.info("assessedUser" + assessedUser) //validateUserCriteria val userDetails = validateUser(assessedUser.userId, criteria.getOrElse(config.user, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], additionalProps)(metrics, config, httpUtil) - + logger.info("userDetails" + userDetails) //generateCertificateEvent if(userDetails.nonEmpty) { generateCertificateEvent(event, template, userDetails, enrolledUser, assessedUser, additionalProps, certName)(metrics, config, cache, httpUtil) From efb558ba49514fc55062f24b6d1b6f60ffc4f13d Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 4 Oct 2023 15:18:34 +0530 Subject: [PATCH 42/88] Added the changes for issueCertificate logic --- .../functions/ProgramCertPreProcessorFn.scala | 29 +++++++++++++++---- .../task/ProgramCertPreProcessorConfig.scala | 1 + 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 6e326cfe2..c91b24292 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -1,6 +1,6 @@ package org.sunbird.job.programcert.functions -import com.datastax.driver.core.Row +import com.datastax.driver.core.{Row, TypeTokens} import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils @@ -72,9 +72,10 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: val courseId: String = childNode.get(config.identifier).asInstanceOf[String] val userId: String = event.userId val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) - val isCertificateIssued: List[Row] = getCourseIssuedCertificateForUser(primaryFields)(metrics) + val isCertificateIssued = getCourseIssuedCertificateForUser(primaryFields)(metrics) + logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) breakable { - if (isCertificateIssued == null || isCertificateIssued.isEmpty) { + if (!isCertificateIssued) { isProgramCertificateToBeGenerated = false; break } @@ -106,6 +107,7 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: "courseId" -> courseParentId)) val requestBody = JSONUtil.serialize(eData) + logger.info("Added the kafka event for programId: " + courseParentId + " requestBody:" + requestBody) context.output(config.generateCertificateOutputTag, requestBody) } } @@ -133,8 +135,9 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: else new java.util.HashMap[String, AnyRef]() } - private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics) = { + private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics): Boolean = { logger.info("primary columns {}", columns) + var isCertificateIssued : Boolean = false; val selectWhere = QueryBuilder.select().all() .from(config.keyspace, config.userEnrolmentsTable). where() @@ -147,7 +150,20 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } }) logger.info("select query {}", selectWhere.toString) - cassandraUtil.find(selectWhere.toString).asScala.toList + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + row = row.filter(x => x.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])) != null).toList + if (row.size() == 1) { + isCertificateIssued = true + } else { + logger.error("More than one certificate" + columns) + isCertificateIssued = false + } + } else { + logger.error("No Certificate Available" + columns) + isCertificateIssued = false + } + isCertificateIssued } def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { @@ -159,13 +175,14 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: metrics.incCounter(config.dbReadCount) var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) if (null != row) { - row = row.filter(x => x.getInt("status") != 2).toList if (row.size() == 1) { row.asScala.get(0) } else { + logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) null } } else { + logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) null } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index b518b1b34..c7934ab21 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -67,5 +67,6 @@ class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJob val userAccBlockedErrCode = "UOS_USRRED0006" val programCertPreProcess: String = "program_cert_pre_process" val parentCollections: String = "parentCollections" + val issuedCertificates: String = "issued_certificates" } From f6987bdfc597817e76200e17b5cf81dab55363da Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 4 Oct 2023 15:30:51 +0530 Subject: [PATCH 43/88] Added the changes for program-activity-aggregator --- program-activity-aggregate-updater/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/pom.xml b/program-activity-aggregate-updater/pom.xml index 167447f4c..a727be571 100644 --- a/program-activity-aggregate-updater/pom.xml +++ b/program-activity-aggregate-updater/pom.xml @@ -159,7 +159,7 @@ - org.sunbird.job.programaggregate.task.ActivityAggregateUpdaterStreamTask + org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterStreamTask Date: Wed, 4 Oct 2023 18:26:55 +0530 Subject: [PATCH 44/88] Enhanced course completion cert request (#32) * Enhanced course completion cert request Enhanced course completion cert request , new parameters added in the email template --- .../functions/IssueCertificateHelper.scala | 55 ++++++++++++++++++- .../CollectionCertPreProcessorConfig.scala | 6 ++ .../sunbird/job/certgen/domain/Event.scala | 5 ++ .../CertificateGeneratorFunction.scala | 3 +- .../certgen/functions/NotifierFunction.scala | 10 +++- .../task/CertificateGeneratorConfig.scala | 4 +- .../certgen/spec/NotifierFunctionTest.scala | 2 +- 7 files changed, 77 insertions(+), 8 deletions(-) diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 3f946a1b5..2a7d26e7e 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -210,10 +210,18 @@ trait IssueCertificateHelper { val lastName = Option(userDetails.getOrElse("lastName", "").asInstanceOf[String]).getOrElse("") def nullStringCheck(name:String):String = {if(StringUtils.equalsIgnoreCase("null", name)) "" else name} val recipientName = nullStringCheck(firstName).concat(" ").concat(nullStringCheck(lastName)).trim - val courseName = getCourseName(event.courseId)(metrics, config, cache, httpUtil) + val courseInfo:java.util.Map[String, AnyRef] = getCourseInfo(event.courseId)(metrics, config, cache, httpUtil) + val courseName = courseInfo.getOrDefault("courseName", "").asInstanceOf[String] val dateFormatter = new SimpleDateFormat("yyyy-MM-dd") val related = getRelatedData(event, enrolledUser, assessedUser, userDetails, additionalProps, certName, courseName)(config) val providerName = getCourseOrganisation(event.courseId)(metrics, config, cache, httpUtil) + val parentCollections: List[String] = Option(courseInfo.get(config.parentCollections)) + .collect { + case list: java.util.List[_] => + list.asInstanceOf[java.util.List[String]].asScala.toList + } + .getOrElse(List.empty) + val eData = Map[String, AnyRef] ( "issuedDate" -> dateFormatter.format(enrolledUser.issuedOn), "data" -> List(Map[String, AnyRef]("recipientName" -> recipientName, "recipientId" -> event.userId)), @@ -230,7 +238,11 @@ trait IssueCertificateHelper { "related" -> related, "name" -> certName, "providerName" -> providerName, - "tag" -> event.batchId + "tag" -> event.batchId, + "primaryCategory" -> courseInfo.getOrDefault("primaryCategory", "").asInstanceOf[String], + "parentCollections" -> parentCollections, + "coursePosterImage" -> courseInfo.getOrDefault("coursePosterImage", "").asInstanceOf[String], + ) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) @@ -252,4 +264,43 @@ trait IssueCertificateHelper { Map[String, Any]("batchId" -> event.batchId, "courseId" -> event.courseId, "type" -> certName) ++ locationProps ++ enrolledUser.additionalProps ++ assessedUser.additionalProps ++ userAdditionalProps ++ courseAdditionalProps } + + def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory,posterImage,organisation" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') + val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + val orgData = response.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length - 1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap + } else { + val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = courseMetadata.getOrElse("parentcollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] + val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse("posterimage", "").asInstanceOf[String]).filter(_ >= ' ') + val orgData = courseMetadata.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length - 1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap + } + } } diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala index f986cfc84..da377789c 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala @@ -79,6 +79,12 @@ class CollectionCertPreProcessorConfig(override val config: Config) extends Base val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") val userAccBlockedErrCode = "UOS_USRRED0006" val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase+ "/content/v3/read/" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val posterImage: String = "posterImage" + val parentCollections: String = "parentCollections" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala index cb08ab00c..c7259302b 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala @@ -60,4 +60,9 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def providerName: String = readOrDefault[String]("edata.providerName", "") + def primaryCategory: String = readOrDefault[String]("edata.primaryCategory", "") + + def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + + def coursePosterImage: String = readOrDefault[String]("edata.coursePosterImage", "") } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index a8d3a92d6..893a98c3c 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -307,7 +307,8 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val audit = ScalaJsonUtil.serialize(certificateAuditEvent) context.output(config.auditEventOutputTag, audit) logger.info("pushAuditEvent: certificate audit event success {}", audit) - context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset)) + context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, + certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { metrics.incCounter(config.failedEventCount) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala index 9711492cf..b4ae85261 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala @@ -20,7 +20,7 @@ import org.sunbird.job.{BaseProcessFunction, Metrics} import scala.collection.JavaConverters._ import scala.collection.mutable -case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long) +case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long, courseProvider: String, coursePosterImage:String) class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[NotificationMetaData, String](config) { @@ -52,7 +52,7 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ val row = getNotificationTemplates(primaryFields, metrics) val certTemplate = row.getMap(config.cert_templates, com.google.common.reflect.TypeToken.of(classOf[String]), TypeTokens.mapOf(classOf[String], classOf[String])) - val url = config.learnerServiceBaseUrl + config.notificationEndPoint + val url = config.learnerServiceBaseUrl + config.newEmailTemplateNotificationEndPoint if (certTemplate != null && StringUtils.isNotBlank(metaData.templateId) && certTemplate.containsKey(metaData.templateId) && certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) { @@ -66,7 +66,11 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ config.heldDate -> dateFormatter.format(metaData.issuedOn), config.recipientUserIds -> List[String](metaData.userId), config.ratingPageUrl -> ratingUrl, - config.body -> "email body"))) + config.body -> "email body", + config.courseName -> metaData.courseName, + config.courseProvider -> metaData.courseProvider, + config.coursePosterImage -> metaData.coursePosterImage + ))) val response = httpUtil.post(url, ScalaJsonUtil.serialize(request)) if (response.status == 200) { diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index e5793226d..1c10b5b64 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -164,5 +164,7 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val userFeedMsg: String = "You have earned a certificate! Download it from your profile page." val priorityValue = 1 val userFeedCount = "user-feed-count" - + val courseProvider: String = "courseProvider" + val coursePosterImage: String = "coursePosterImage" + val newEmailTemplateNotificationEndPoint: String = "/v1/notification/email" } diff --git a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala index 2cd445226..7d63ef1f2 100755 --- a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala +++ b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala @@ -64,7 +64,7 @@ class NotifierFunctionTest extends BaseTestSpec { "NotifierFunction " should "should send notify user" in { implicit val notificationMetaTypeInfo: TypeInformation[NotificationMetaData] = TypeExtractor.getForClass(classOf[NotificationMetaData]) - new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0), null, metrics) + new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0,"", ""), null, metrics) metrics.get(s"${notifierConfig.courseBatchdbReadCount}") should be(1) metrics.get(s"${notifierConfig.notifiedUserCount}") should be(1) metrics.get(s"${notifierConfig.skipNotifyUserCount}") should be(0) From b71e4f85eef95035d218b549345ac9ee15927ac5 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 5 Oct 2023 09:41:32 +0530 Subject: [PATCH 45/88] Added the changes for program-activity-aggregator --- .../task/ProgramActivityAggregateUpdaterStreamTask.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index 9c196882a..b21ba153d 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -27,7 +27,7 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate .uid(config.activityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance .getSideOutput(config.uniqueConsumptionOutput) - .keyBy(new ActivityAggregatorKeySelector(config)) + .keyBy(new ProgramActivityAggregatorKeySelector(config)) .countWindow(config.thresholdBatchReadSize) .process(new ProgramActivityAggregatesFunction(config, httpUtil)) .name(config.programactivityAggregateUpdaterFn) @@ -45,7 +45,7 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate } // $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster -object ActivityAggregateUpdaterStreamTask { +object ProgramActivityAggregateUpdaterStreamTask { def main(args: Array[String]): Unit = { val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) @@ -62,7 +62,7 @@ object ActivityAggregateUpdaterStreamTask { } // $COVERAGE-ON$ -class ActivityAggregatorKeySelector(config: ProgramActivityAggregateUpdaterConfig) extends KeySelector[Map[String, AnyRef], Int] { +class ProgramActivityAggregatorKeySelector(config: ProgramActivityAggregateUpdaterConfig) extends KeySelector[Map[String, AnyRef], Int] { private val serialVersionUID = 7267989625042068736L private val shards = config.windowShards override def getKey(in: Map[String, AnyRef]): Int = { From 00d2b91feb35e9731c995c44afa77292657256c5 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 5 Oct 2023 11:00:38 +0530 Subject: [PATCH 46/88] Added the changes for program-activity-aggregator Config fix --- .../main/resources/program-activity-aggregate-updater.conf | 5 ++--- .../task/ProgramActivityAggregateUpdaterConfig.scala | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf index 9f8d79d13..316d3b385 100644 --- a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf +++ b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf @@ -48,7 +48,6 @@ activity { } service { - search { - basePath = "http://11.2.6.6/search" - } + content.basePath = "http://11.2.6.6/content" + search.basePath = "http://11.2.6.6/search" } \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala index 6bcb344c2..05b17d604 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -9,7 +9,7 @@ import org.sunbird.job.programaggregate.domain.CollectionProgress import java.util -class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "activity-aggregate-updater") { +class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "program-activity-aggregate-updater") { private val serialVersionUID = 2905979434303791379L @@ -133,7 +133,7 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val searchAPIURL = searchServiceBasePath + "/v3/search" val contentServiceBase: String = config.getString("service.content.basePath") - val contentReadURL = contentServiceBase + "/content/v3/read/" + val contentReadURL = contentServiceBase + "/v3/read/" val identifier: String = "identifier" val primaryCategory: String = "primaryCategory" From 1d2a962940ab841118c087abca78761dec656309 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 5 Oct 2023 13:05:26 +0530 Subject: [PATCH 47/88] Added the config changes for program-activity-aggregator --- .../main/resources/program-activity-aggregate-updater.conf | 2 +- .../task/ProgramActivityAggregateUpdaterConfig.scala | 2 +- .../task/ProgramActivityAggregateUpdaterStreamTask.scala | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf index 316d3b385..a51a37d1d 100644 --- a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf +++ b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf @@ -5,7 +5,7 @@ kafka { output.audit.topic = "sunbirddev.telemetry.raw" output.failed.topic = "sunbirddev.activity.agg.failed" output.certissue.topic = "sunbirddev.issue.certificate.request" - groupId = "sunbirddev-activity-aggregate-updater-group" + groupId = "sunbirddev-program-activity-aggregate-updater-group" } task { diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala index 05b17d604..365dd6c1d 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -107,7 +107,7 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val aggregates = "aggregates" // Consumers - val activityAggregateUpdaterConsumer = "activity-aggregate-updater-consumer" + val programActivityAggregateUpdaterConsumer = "program-activity-aggregate-updater-consumer" // Producers val programactivityAggregateUpdaterProducer = "program-activity-aggregate-updater-audit-events-sink" diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index b21ba153d..02f2a0a57 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -23,8 +23,8 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]] = TypeExtractor.getForClass(classOf[List[CollectionProgress]]) val progressStream = - env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.activityAggregateUpdaterConsumer) - .uid(config.activityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) + env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.programActivityAggregateUpdaterConsumer) + .uid(config.programActivityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance .getSideOutput(config.uniqueConsumptionOutput) .keyBy(new ProgramActivityAggregatorKeySelector(config)) From b1831945e30b5f738f31ae03a5c70966fea4c61c Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 06:12:08 +0530 Subject: [PATCH 48/88] Updated program activity aggregate updater flink job --- .../ContentConsumptionDeDupFunction.scala | 4 +- program-activity-aggregate-updater/pom.xml | 472 +++++++++--------- .../programaggregate/common/DeDupHelper.scala | 2 +- ...ogramContentConsumptionDeDupFunction.scala | 74 +++ .../ProgramProgressCompleteFunction.scala | 2 +- .../ProgramProgressUpdateFunction.scala | 2 +- 6 files changed, 316 insertions(+), 240 deletions(-) create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala index 1ee934f2f..abe0290bb 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala @@ -18,10 +18,10 @@ import org.sunbird.job.aggregate.task.ActivityAggregateUpdaterConfig import scala.collection.JavaConverters._ -class ContentConsumptionDeDupFunction(config: ActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { +class ContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType - private[this] val logger = LoggerFactory.getLogger(classOf[ContentConsumptionDeDupFunction]) + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramContentConsumptionDeDupFunction]) var deDupEngine: DeDupEngine = _ override def open(parameters: Configuration): Unit = { diff --git a/program-activity-aggregate-updater/pom.xml b/program-activity-aggregate-updater/pom.xml index a727be571..e5d97f18a 100644 --- a/program-activity-aggregate-updater/pom.xml +++ b/program-activity-aggregate-updater/pom.xml @@ -1,242 +1,244 @@ - - - 4.0.0 - - org.sunbird - knowledge-platform-jobs - 1.0 - - org.sunbird - program-activity-aggregate-updater - 1.0.0 - jar - program-activity-aggregate-updater - - Program Progress Computation - + - - UTF-8 - 1.4.0 - + + 4.0.0 + + org.sunbird + knowledge-platform-jobs + 1.0 + + program-activity-aggregate-updater + 1.0.0 + jar + program-activity-aggregate-updater + + Program Progress Computation + + + + UTF-8 + 1.4.0 + - - - org.apache.flink - flink-streaming-scala_${scala.version} - ${flink.version} - provided - - - org.sunbird - jobs-core - 1.0.0 - - - joda-time - joda-time - 2.10.6 - - - com.twitter - storehaus-cache_${scala.version} - 0.15.0 - - - org.sunbird - jobs-core - 1.0.0 - test-jar - test - - - org.apache.flink - flink-test-utils_${scala.version} - ${flink.version} - test - - - org.apache.flink - flink-runtime_${scala.version} - ${flink.version} - test - tests - - - it.ozimov - embedded-redis - 0.7.1 - test - - - org.apache.flink - flink-streaming-java_${scala.version} - ${flink.version} - test - tests - - - org.scalatest - scalatest_${scala.version} - 3.0.6 - test - - - org.mockito - mockito-core - 3.3.3 - test - - - com.fiftyonred - mock-jedis - 0.4.0 - test - - - org.cassandraunit - cassandra-unit - 3.11.2.0 - test - - - it.ozimov - embedded-redis - 0.7.1 - test - - - com.google.guava - guava - - - - + + + org.apache.flink + flink-streaming-scala_${scala.version} + ${flink.version} + provided + + + org.sunbird + jobs-core + 1.0.0 + + + joda-time + joda-time + 2.10.6 + + + com.twitter + storehaus-cache_${scala.version} + 0.15.0 + + + org.sunbird + jobs-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_${scala.version} + ${flink.version} + test + + + org.apache.flink + flink-runtime_${scala.version} + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + org.apache.flink + flink-streaming-java_${scala.version} + ${flink.version} + test + tests + + + org.scalatest + scalatest_${scala.version} + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + it.ozimov + embedded-redis + 0.7.1 + test + + + com.google.guava + guava + + + + - - src/main/scala - src/test/scala - - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - 11 - - + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + - - org.apache.maven.plugins - maven-shade-plugin - 3.2.1 - - - - package - - shade - - - - - com.google.code.findbugs:jsr305 - - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterStreamTask - - - - reference.conf - - - - - - + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterStreamTask + + + + reference.conf + + + + + + - - net.alchim31.maven - scala-maven-plugin - 4.4.0 - - 11 - 11 - ${scala.maj.version} - false - - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + 11 + 11 + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + - - maven-surefire-plugin - 2.22.2 - - true - - + + maven-surefire-plugin + 2.22.2 + + true + + - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - dp-duplication-testsuite.txt - - - - test - - test - - - - - - org.scoverage - scoverage-maven-plugin - ${scoverage.plugin.version} - - ${scala.version} - true - true - - - - - + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + dp-duplication-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala index 3bce467e3..fda596c54 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala @@ -1,4 +1,4 @@ -package org.sunbird.job.aggregate.common +package org.sunbird.job.programaggregate.common import java.security.MessageDigest diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala new file mode 100644 index 000000000..8b08cda3b --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -0,0 +1,74 @@ +package org.sunbird.job.programaggregate.functions + +import java.lang.reflect.Type +import java.security.MessageDigest +import java.util + +import com.google.gson.reflect.TypeToken +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.programaggregate.common.DeDupHelper +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.{BaseProcessFunction, Metrics} +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig + +import scala.collection.JavaConverters._ + +class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { + + val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramContentConsumptionDeDupFunction]) + var deDupEngine: DeDupEngine = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + deDupEngine.close() + super.close() + } + + override def processElement(event: util.Map[String, AnyRef], context: ProcessFunction[util.Map[String, AnyRef], String]#Context, metrics: Metrics): Unit = { + metrics.incCounter(config.totalEventCount) + val eData = event.get(config.eData).asInstanceOf[util.Map[String, AnyRef]].asScala + val isBatchEnrollmentEvent: Boolean = StringUtils.equalsIgnoreCase(eData.getOrElse(config.action, "").asInstanceOf[String], config.batchEnrolmentUpdateCode) + if (isBatchEnrollmentEvent) { + val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala + val filteredContents = contents.filter(x => x.get("status") == 2).toList + if (filteredContents.size == 0) + metrics.incCounter(config.skipEventsCount) + else + metrics.incCounter(config.batchEnrolmentUpdateEventCount) + filteredContents.map(c => { + (eData + ("contents" -> List(Map("contentId" -> c.get("contentId"), "status" -> c.get("status"))))).toMap + }).filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d)) + } else metrics.incCounter(config.skipEventsCount) + } + + override def metricsList(): List[String] = { + List(config.totalEventCount, config.skipEventsCount, config.batchEnrolmentUpdateEventCount) + } + + def discardDuplicates(event: Map[String, AnyRef]): Boolean = { + if (config.dedupEnabled) { + val userId = event.getOrElse(config.userId, "").asInstanceOf[String] + val courseId = event.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId = event.getOrElse(config.batchId, "").asInstanceOf[String] + val contents = event.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] + if (contents.nonEmpty) { + val content = contents.head + val contentId = content.getOrElse("contentId", "").asInstanceOf[String] + val status = content.getOrElse("status", 0.asInstanceOf[AnyRef]).asInstanceOf[Number].intValue() + val checksum = DeDupHelper.getMessageId(courseId, batchId, userId, contentId, status) + deDupEngine.isUniqueEvent(checksum) + } else false + } else true + } +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala index 4bb6aa2a8..cd8cbd385 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala @@ -9,7 +9,7 @@ import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory import org.sunbird.job.cache.RedisConnect -import org.sunbird.job.aggregate.common.DeDupHelper +import org.sunbird.job.programaggregate.common.DeDupHelper import org.sunbird.job.dedup.DeDupEngine import org.sunbird.job.{BaseProcessFunction, Metrics} import org.sunbird.job.programaggregate.domain.{ActorObject, CollectionProgress, EventContext, EventData, EventObject, TelemetryEvent} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala index fbc2cc45f..418c86473 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala @@ -6,7 +6,7 @@ import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory import org.sunbird.job.cache.RedisConnect -import org.sunbird.job.aggregate.common.DeDupHelper +import org.sunbird.job.programaggregate.common.DeDupHelper import org.sunbird.job.dedup.DeDupEngine import org.sunbird.job.programaggregate.domain._ import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig From 982ac6e5e96f16dc0bcc1fc670aec804f7e0bef1 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 06:21:48 +0530 Subject: [PATCH 49/88] Fixed build issues --- .../aggregate/functions/ContentConsumptionDeDupFunction.scala | 4 ++-- .../task/ProgramActivityAggregateUpdaterStreamTask.scala | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala index abe0290bb..1ee934f2f 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ContentConsumptionDeDupFunction.scala @@ -18,10 +18,10 @@ import org.sunbird.job.aggregate.task.ActivityAggregateUpdaterConfig import scala.collection.JavaConverters._ -class ContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { +class ContentConsumptionDeDupFunction(config: ActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType - private[this] val logger = LoggerFactory.getLogger(classOf[ProgramContentConsumptionDeDupFunction]) + private[this] val logger = LoggerFactory.getLogger(classOf[ContentConsumptionDeDupFunction]) var deDupEngine: DeDupEngine = _ override def open(parameters: Configuration): Unit = { diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index 02f2a0a57..9b3cc6652 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -8,7 +8,7 @@ import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.streaming.api.scala._ import org.sunbird.job.programaggregate.domain.CollectionProgress import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesFunction} +import org.sunbird.job.programaggregate.functions.{ProgramContentConsumptionDeDupFunction, ProgramActivityAggregatesFunction} import org.sunbird.job.util.{FlinkUtil, HttpUtil} import java.io.File @@ -26,6 +26,8 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.programActivityAggregateUpdaterConsumer) .uid(config.programActivityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance + .process(new ProgramContentConsumptionDeDupFunction(config)).name(config.consumptionDeDupFn) + .uid(config.consumptionDeDupFn).setParallelism(config.deDupProcessParallelism) .getSideOutput(config.uniqueConsumptionOutput) .keyBy(new ProgramActivityAggregatorKeySelector(config)) .countWindow(config.thresholdBatchReadSize) From f976b184e12bddb60e53389049c1e9b4e7ce61d1 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 12:09:07 +0530 Subject: [PATCH 50/88] Updated config names --- .../task/ProgramActivityAggregateUpdaterConfig.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala index 365dd6c1d..b4e24acc9 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -59,17 +59,17 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val deDupExpirySec: Int = config.getInt("dedup-redis.database.expiry") // Tags - val uniqueConsumptionOutputTagName = "unique-consumption-events" + val uniqueConsumptionOutputTagName = "program-unique-consumption-events" val uniqueConsumptionOutput: OutputTag[Map[String, AnyRef]] = OutputTag[Map[String, AnyRef]](uniqueConsumptionOutputTagName) val auditEventOutputTagName = "audit-events" val auditEventOutputTag: OutputTag[String] = OutputTag[String](auditEventOutputTagName) val failedEventOutputTagName = "failed-events" val failedEventOutputTag: OutputTag[String] = OutputTag[String](failedEventOutputTagName) - val collectionCompleteOutputTagName = "collection-progress-complete-events" + val collectionCompleteOutputTagName = "program-collection-progress-complete-events" val collectionCompleteOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionCompleteOutputTagName) - val collectionUpdateOutputTagName = "collection-progress-update-events" + val collectionUpdateOutputTagName = "program-collection-progress-update-events" val collectionUpdateOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionUpdateOutputTagName) - val certIssueOutputTagName = "certificate-issue-events" + val certIssueOutputTagName = "program-certificate-issue-events" val certIssueOutputTag: OutputTag[String] = OutputTag[String](certIssueOutputTagName) // constants @@ -98,7 +98,7 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val action = "action" val batchEnrolmentUpdateCode = "batch-enrolment-update" val routerFn = "RouterFn" - val consumptionDeDupFn= "consumption-dedup-process" + val consumptionDeDupFn= "program-consumption-dedup-process" val programactivityAggregateUpdaterFn = "program-activity-aggregate-updater-fn" val partition = "partition" val courseBatch = "CourseBatch" From 41d32626d789e41a292cc6c3c15eb2f891098dac Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 14:52:18 +0530 Subject: [PATCH 51/88] Added logs --- .../job/aggregate/functions/ActivityAggregatesFunction.scala | 1 + .../functions/ProgramActivityAggregatesFunction.scala | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index 3e36375c9..576f4da52 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -471,6 +471,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti } else { logger.error("Failed to read content details for Id: " + identifier) } + logger.info("is activity aggregator is skipping this event ? " + isValidProgram) isValidProgram } diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 49e2ba5bb..c656aa78a 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -473,7 +473,8 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) } else { logger.error("Failed to read content details for Id: " + identifier) - } + } + logger.info("is program activity aggregator is skipping this event ? " + isValidProgram) isValidProgram } From c8a3bb0330f797eccc0e08516e5dd167e620d34b Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 6 Oct 2023 15:47:19 +0530 Subject: [PATCH 52/88] Dev #107756 Code Review changes for Flink-job for program-cert-pre-processor --- .../functions/ProgramCertPreProcessorFn.scala | 349 ++++++++++-------- .../task/ProgramCertPreProcessorConfig.scala | 120 +++--- 2 files changed, 262 insertions(+), 207 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index c91b24292..974c72106 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -1,7 +1,8 @@ package org.sunbird.job.programcert.functions import com.datastax.driver.core.{Row, TypeTokens} -import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.common.reflect.TypeToken import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.typeinfo.TypeInformation @@ -16,174 +17,224 @@ import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} +import java.util.UUID import scala.collection.JavaConverters._ import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} +import scala.collection.mutable import scala.util.control.Breaks.{break, breakable} class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil) (implicit val stringTypeInfo: TypeInformation[String], - @transient var cassandraUtil: CassandraUtil = null) + @transient var cassandraUtil: CassandraUtil = null) extends BaseProcessKeyedFunction[String, Event, String](config) with IssueCertificateHelper { - private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) - private var cache: DataCache = _ - private var contentCache: DataCache = _ - lazy private val mapper: ObjectMapper = new ObjectMapper() - - override def open(parameters: Configuration): Unit = { - super.open(parameters) - cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) - val redisConnect = new RedisConnect(config) - cache = new DataCache(config, redisConnect, config.collectionCacheStore, List()) - cache.init() - - val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) - contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) - contentCache.init() - } + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) + private var cache: DataCache = _ + private var contentCache: DataCache = _ + lazy private val mapper: ObjectMapper = new ObjectMapper() - override def close(): Unit = { - cassandraUtil.close() - cache.close() - super.close() - } + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + val redisConnect = new RedisConnect(config) + cache = new DataCache(config, redisConnect, config.collectionCacheStore, List()) + cache.init() - override def metricsList(): List[String] = { - List(config.totalEventsCount, config.dbReadCount, config.dbUpdateCount, config.failedEventCount, config.skippedEventCount, config.successEventCount, - config.cacheHitCount) - } + val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) + contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) + contentCache.init() + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + override def metricsList(): List[String] = { + List(config.totalEventsCount, config.dbReadCount, config.dbUpdateCount, config.failedEventCount, config.skippedEventCount, config.successEventCount, + config.cacheHitCount) + } + + override def processElement(event: Event, + context: KeyedProcessFunction[String, Event, String]#Context, + metrics: Metrics): Unit = { + try { + val getParentIdForCourse = event.parentCollections + if (!getParentIdForCourse.isEmpty) { + for (courseParentId <- getParentIdForCourse) { + val programEnrollmentStatus = getEnrolment(event.userId, courseParentId)(metrics) + //if enrolled into program + if (null != programEnrollmentStatus && programEnrollmentStatus.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty) { + val programHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) + if (!programHierarchy.isEmpty) { + val batchId: String = programEnrollmentStatus.getString(config.dbBatchId) + val contentDataForProgram = programHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] + val leafNodeMap = mutable.Map[String, Int]() - override def processElement(event: Event, - context: KeyedProcessFunction[String, Event, String]#Context, - metrics: Metrics): Unit = { - try { - val getParentIdForCourse = event.parentCollections - if (!getParentIdForCourse.isEmpty) { - for (courseParentId <- getParentIdForCourse) { - val programEnrollmentStatus = getEnrolment(event.userId, courseParentId)(metrics) - //if enrolled into program - if (null != programEnrollmentStatus) { - val curatedProgramHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) - if (!curatedProgramHierarchy.isEmpty) { - val batchId: String = programEnrollmentStatus.getString(config.dbBatchId) - val contentDataForProgram = curatedProgramHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] - var isProgramCertificateToBeGenerated: Boolean = true; - for (childNode <- contentDataForProgram) { - val courseId: String = childNode.get(config.identifier).asInstanceOf[String] - val userId: String = event.userId - val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) - val isCertificateIssued = getCourseIssuedCertificateForUser(primaryFields)(metrics) - logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) - breakable { - if (!isCertificateIssued) { - isProgramCertificateToBeGenerated = false; - break - } - } - } - if (isProgramCertificateToBeGenerated) { - //Add kafka event to generate Certificate for Program - val eData = Map[String, AnyRef]( - "eid" -> "BE_JOB_REQUEST", - "ets" -> "1695019553226", - "mid" -> "LP.1695019553226.dcfd4458-f7e4-4e23-bfb7-9531ec91a1fe", - "actor" -> Map( - "id" -> "Program Certificate Generator", - "type" -> "System"), - "context" -> Map( - "pdata" -> Map( - "ver" -> "1.0", - "id" -> "org.sunbird.platform")), - "object" -> Map( - "id" -> batchId.concat("_").concat(courseParentId), - "type" -> "ProgramCertificateGeneration"), - "edata" -> Map( - "userIds" -> List(event.userId), - "action" -> "issue-certificate", - "iteration" -> 1, - "trigger" -> "auto-issue", - "batchId" -> batchId, - "reIssue" -> false, - "courseId" -> courseParentId)) - - val requestBody = JSONUtil.serialize(eData) - logger.info("Added the kafka event for programId: " + courseParentId + " requestBody:" + requestBody) - context.output(config.generateCertificateOutputTag, requestBody) - } - } + var isProgramCertificateToBeGenerated: Boolean = true; + for (childNode <- contentDataForProgram) { + val primaryCategory = childNode.get(config.primaryCategory).asInstanceOf[String] + if (config.allowedPrimaryCategoryForProgram.contains(primaryCategory)) { + val courseId: String = childNode.get(config.identifier).asInstanceOf[String] + val userId: String = event.userId + val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) + val isCertificateIssued = getCourseIssuedCertificateForUser(primaryFields)(metrics) + logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) + breakable { + if (!isCertificateIssued) { + isProgramCertificateToBeGenerated = false; + break + } else { + val leafNodes = childNode.get(config.leafNodes).asInstanceOf[java.util.List[String]] + for (leafNode <- leafNodes) { + leafNodeMap += (leafNode -> 2) + } } + } } + } + if (!leafNodeMap.isEmpty) { + val programContentStatus = Option(programEnrollmentStatus.getMap( + config.contentStatus, TypeToken.of(classOf[String]), TypeToken.of(classOf[Integer]))).head + var progressCount: Integer = Option(programEnrollmentStatus.getInt(config.progress)).head + + var updateCount = 0 + + for ((key, value) <- leafNodeMap) { + // Check if the key is present in leafNodeMap + if (programContentStatus.get(key) != null) { + if (programContentStatus.get(key) != 2) { + // Update progress in contentStatus for the matching key + programContentStatus.put(key, value) + updateCount += 1 + } + } else { + programContentStatus.put(key, value) + updateCount += 1 + } + } + + // Update the progress with the total update count + progressCount += updateCount + var status: Int = 1 + val leafNodesForProgram = programHierarchy.get(config.leafNodes).asInstanceOf[java.util.List[String]] + if (progressCount == leafNodesForProgram.size()) { + status = 2 + } else { + isProgramCertificateToBeGenerated = false + } + updateEnrolment(event.userId, batchId, courseParentId, programContentStatus, status, progressCount)(metrics) + } + + if (isProgramCertificateToBeGenerated) { + //Add kafka event to generate Certificate for Program + logger.info("Adding the kafka event for programId: " + courseParentId) + createIssueCertEventForProgram(courseParentId, event.userId, batchId, context)(metrics) + } } - } catch { - case ex: Exception => { - throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) - } + } } - logger.info("Inside the Process ElementForProgram"); + } + } catch { + case ex: Exception => { + throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) + } } + logger.info("Inside the Process ElementForProgram"); + } - def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { - val query = QueryBuilder.select(config.Hierarchy).from(config.contentHierarchyKeySpace, config.contentHierarchyTable) - .where(QueryBuilder.eq(config.identifier, programId)) - val row = cassandraUtil.find(query.toString) - if (CollectionUtils.isNotEmpty(row)) { - val hierarchy = row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] - if (StringUtils.isNotBlank(hierarchy)) - mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) - else new java.util.HashMap[String, AnyRef]() - } - else new java.util.HashMap[String, AnyRef]() + def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { + val query = QueryBuilder.select(config.Hierarchy).from(config.contentHierarchyKeySpace, config.contentHierarchyTable) + .where(QueryBuilder.eq(config.identifier, programId)) + val row = cassandraUtil.find(query.toString) + if (CollectionUtils.isNotEmpty(row)) { + val hierarchy = row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] + if (StringUtils.isNotBlank(hierarchy)) + mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) + else new java.util.HashMap[String, AnyRef]() } + else new java.util.HashMap[String, AnyRef]() + } - private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics): Boolean = { - logger.info("primary columns {}", columns) - var isCertificateIssued : Boolean = false; - val selectWhere = QueryBuilder.select().all() - .from(config.keyspace, config.userEnrolmentsTable). - where() - columns.map(col => { - col._2 match { - case value: List[Any] => - selectWhere.and(QueryBuilder.in(col._1, value.asJava)) - case _ => - selectWhere.and(QueryBuilder.eq(col._1, col._2)) - } - }) - logger.info("select query {}", selectWhere.toString) - var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) - if (null != row) { - row = row.filter(x => x.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])) != null).toList - if (row.size() == 1) { - isCertificateIssued = true - } else { - logger.error("More than one certificate" + columns) - isCertificateIssued = false - } - } else { - logger.error("No Certificate Available" + columns) - isCertificateIssued = false - } - isCertificateIssued + private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics): Boolean = { + logger.info("primary columns {}", columns) + var isCertificateIssued: Boolean = false; + val selectWhere = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + columns.map(col => { + col._2 match { + case value: List[Any] => + selectWhere.and(QueryBuilder.in(col._1, value.asJava)) + case _ => + selectWhere.and(QueryBuilder.eq(col._1, col._2)) + } + }) + logger.info("select query {}", selectWhere.toString) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + row = row.filter(x => x.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])) != null).toList + if (row.size() == 1) { + isCertificateIssued = true + } else { + logger.error("More than one certificate" + columns) + isCertificateIssued = false + } + } else { + logger.error("No Certificate Available" + columns) + isCertificateIssued = false } + isCertificateIssued + } - def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { - val selectWhere: Select.Where = QueryBuilder.select().all() - .from(config.keyspace, config.userEnrolmentsTable). - where() - selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) - .and(QueryBuilder.eq(config.dbCourseId, programId)) - metrics.incCounter(config.dbReadCount) - var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) - if (null != row) { - if (row.size() == 1) { - row.asScala.get(0) - } else { - logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) - null - } - } else { - logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) - null - } + def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) + .and(QueryBuilder.eq(config.dbCourseId, programId)) + metrics.incCounter(config.dbReadCount) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + if (row.size() == 1) { + row.asScala.get(0) + } else { + logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) + null + } + } else { + logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) + null } + } + + def updateEnrolment(userId: String, batchId: String, programId: String, contentStatus: java.util.Map[String, Integer], status: Int, progress: Int)(implicit metrics: Metrics): Unit = { + logger.info("Enrolment updated for userId: " + userId + " batchId: " + batchId) + val updateQuery = QueryBuilder.update(config.keyspace, config.userEnrolmentsTable) + .`with`(QueryBuilder.set("status", status)) + .and(QueryBuilder.set("progress", progress)) + .and(QueryBuilder.set("contentstatus", contentStatus)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + .where(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", programId)) + .and(QueryBuilder.eq("batchid", batchId)) + val result = cassandraUtil.upsert(updateQuery.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed" + updateQuery.toString + logger.error(msg) + throw new Exception(msg) + } + } + + def createIssueCertEventForProgram(programId: String, userId: String, batchId: String, context: KeyedProcessFunction[String, Event, String]#Context)(implicit metrics: Metrics): Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Program Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${batchId}_${programId}","type": "ProgramCertificateGeneration"},"edata": {"userIds": ["${userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${batchId}","reIssue": false,"courseId": "${programId}"}}""" + logger.info("o/p event: " + event) + context.output(config.generateCertificateOutputTag, event) + metrics.incCounter(config.programCertIssueEventsCount) + } } diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala index c7934ab21..f817f773d 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -6,67 +6,71 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.scala.OutputTag import org.sunbird.job.BaseJobConfig -import java.util - class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJobConfig(config, "program-cert-pre-processor") { - implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) - - //Redis config - val collectionCacheStore: Int = 0 - val contentCacheStore: Int = 5 - val metaRedisHost: String = config.getString("redis-meta.host") - val metaRedisPort: Int = config.getInt("redis-meta.port") - - //kafka config - val kafkaInputTopic: String = config.getString("kafka.input.topic") - val kafkaOutputTopic: String = config.getString("kafka.output.topic") - val certificatePreProcessorConsumer: String = "program-cert-pre-processor-consumer" - val generateCertificateProducer = "generate-certificate-sink" - override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") - val generateCertificateParallelism:Int = config.getInt("task.generate_certificate.parallelism") - - //Tags - val generateCertificateOutputTagName = "generate-certificate-request" - val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + + //Redis config + val collectionCacheStore: Int = 0 + val contentCacheStore: Int = 5 + val metaRedisHost: String = config.getString("redis-meta.host") + val metaRedisPort: Int = config.getInt("redis-meta.port") + + //kafka config + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaOutputTopic: String = config.getString("kafka.output.topic") + val certificatePreProcessorConsumer: String = "program-cert-pre-processor-consumer" + val generateCertificateProducer = "generate-certificate-sink" + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val generateCertificateParallelism: Int = config.getInt("task.generate_certificate.parallelism") + + //Tags + val generateCertificateOutputTagName = "generate-certificate-request" + val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) + + //Cassandra config + val dbHost: String = config.getString("lms-cassandra.host") + val dbPort: Int = config.getInt("lms-cassandra.port") + val keyspace: String = config.getString("lms-cassandra.keyspace") + val userEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") + val dbBatchId = "batchId" + val dbCourseId = "courseid" + val dbUserId = "userid" + val contentHierarchyTable: String = "content_hierarchy" + val contentHierarchyKeySpace: String = "dev_hierarchy_store" + val Hierarchy: String = "hierarchy" + val childrens: String = "children" + val batches: String = "batches" + + //API URL + val contentBasePath = config.getString("service.content.basePath") + val learnerBasePath = config.getString("service.learner.basePath") + val userReadApi = config.getString("user_read_api") + val contentReadApi = "/content/v4/read" - //Cassandra config - val dbHost: String = config.getString("lms-cassandra.host") - val dbPort: Int = config.getInt("lms-cassandra.port") - val keyspace: String = config.getString("lms-cassandra.keyspace") - val userEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") - val dbBatchId = "batchId" - val dbCourseId = "courseid" - val dbUserId = "userid" - val contentHierarchyTable: String = "content_hierarchy" - val contentHierarchyKeySpace: String = "dev_hierarchy_store" - val Hierarchy: String = "hierarchy" - val childrens: String = "children" - val batches: String = "batches" - - //API URL - val contentBasePath = config.getString("service.content.basePath") - val learnerBasePath = config.getString("service.learner.basePath") - val userReadApi = config.getString("user_read_api") - val contentReadApi = "/content/v4/read" + // Metric List + val totalEventsCount = "total-events-count" + val successEventCount = "success-events-count" + val failedEventCount = "failed-events-count" + val skippedEventCount = "skipped-event-count" + val dbReadCount = "db-read-count" + val dbUpdateCount = "db-update-count" + val cacheHitCount = "cache-hit-cout" + val programCertIssueEventsCount = "program-cert-issue-events-count" - // Metric List - val totalEventsCount = "total-events-count" - val successEventCount = "success-events-count" - val failedEventCount = "failed-events-count" - val skippedEventCount = "skipped-event-count" - val dbReadCount = "db-read-count" - val dbUpdateCount = "db-update-count" - val cacheHitCount = "cache-hit-cout" - - //Constants - val status: String = "status" - val name: String = "name" - val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") - val identifier: String = "identifier" - val userAccBlockedErrCode = "UOS_USRRED0006" - val programCertPreProcess: String = "program_cert_pre_process" - val parentCollections: String = "parentCollections" - val issuedCertificates: String = "issued_certificates" + //Constants + val status: String = "status" + val name: String = "name" + val defaultHeaders = Map[String, String]("Content-Type" -> "application/json") + val identifier: String = "identifier" + val userAccBlockedErrCode = "UOS_USRRED0006" + val programCertPreProcess: String = "program_cert_pre_process" + val parentCollections: String = "parentCollections" + val issuedCertificates: String = "issued_certificates" + val primaryCategory: String = "primaryCategory" + val leafNodes: String = "leafNodes" + val contentStatus: String = "contentstatus" + val progress: String = "progress" + val allowedPrimaryCategoryForProgram = List[String]("Course") } From dde37348d48985006478d4e07ff9ffa7e50ac4f0 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 17:59:47 +0530 Subject: [PATCH 53/88] Added type casting --- .../aggregate/functions/ActivityAggregatesFunction.scala | 2 +- .../functions/ProgramActivityAggregatesFunction.scala | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index 576f4da52..41962a178 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -59,7 +59,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti logger.info("Input Events : " + JSONUtil.serialize(events.toList)) val inputUserConsumptionList: List[UserContentConsumption] = events - .filter(event=> verifyPrimaryCategory(event.get(config.courseId).asInstanceOf[String])(metrics, config, httpUtil, cache)) + .filter(event=> verifyPrimaryCategory(event.getOrElse(config.courseId, "").asInstanceOf[String])(metrics, config, httpUtil, cache)) .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { metrics.incCounter(config.processedEnrolmentCount) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index c656aa78a..4390bb946 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -565,9 +565,9 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC cache: DataCache ): Map[String, AnyRef] = { var eventInfoMap: Map[String, AnyRef] = mutable.Map.empty() - var courseId: String = eventData.get(config.courseId).asInstanceOf[String] - var batchId: String = eventData.get(config.batchId).asInstanceOf[String] - var userId: String = eventData.get(config.userId).asInstanceOf[String] + val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] + val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] if ((verifyPrimaryCategory(courseId)(metrics, config, httpUtil, cache))) { eventInfoMap ++= eventData if (StringUtils.isEmpty(batchId)) { From 4a260d0f2447d3f14548bc2d2e34f768b51b670a Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Fri, 6 Oct 2023 18:56:37 +0530 Subject: [PATCH 54/88] Fix for type casting --- .../ProgramActivityAggregatesFunction.scala | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 4390bb946..4eb23cfe4 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -47,8 +47,12 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } override def close(): Unit = { - cassandraUtil.close() - cache.close() + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } super.close() } @@ -564,7 +568,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC httpUtil: HttpUtil, cache: DataCache ): Map[String, AnyRef] = { - var eventInfoMap: Map[String, AnyRef] = mutable.Map.empty() + var eventInfoMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] @@ -578,7 +582,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC return null; } } - eventInfoMap + eventInfoMap.toMap } else { null; } From 7be2af061db8259055abf57159364d07685d7527 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Sat, 7 Oct 2023 13:11:45 +0530 Subject: [PATCH 55/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramActivityAggregatesFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 4eb23cfe4..113813cc2 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -491,7 +491,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val courseMetadata = cache.getWithRetry(courseId) if (null == courseMetadata || courseMetadata.isEmpty) { val url = - config.contentReadURL + "/" + courseId + "?fields=identifier,name,primaryCategory" + config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory" val response = getAPICall(url, "content")(config, httpUtil, metrics) val courseName = StringContext .processEscapes( From f2aa71581572e13c60ece60300a47c8d402aea2a Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Mon, 9 Oct 2023 11:42:02 +0530 Subject: [PATCH 56/88] Added the changes for program-activity-aggregator code fix --- .../task/ProgramActivityAggregateUpdaterConfig.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala index b4e24acc9..54afcfc9a 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -133,7 +133,7 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val searchAPIURL = searchServiceBasePath + "/v3/search" val contentServiceBase: String = config.getString("service.content.basePath") - val contentReadURL = contentServiceBase + "/v3/read/" + val contentReadURL = contentServiceBase + "/content/v3/read/" val identifier: String = "identifier" val primaryCategory: String = "primaryCategory" From aa96b070bfb9e2e3716ad4fcfcc8b51ee3d854c6 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Mon, 9 Oct 2023 12:42:11 +0530 Subject: [PATCH 57/88] Added the changes for program-cert-pre-aggregator Add metrics parama --- .../job/programcert/functions/ProgramCertPreProcessorFn.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 974c72106..bcd935b7e 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -53,7 +53,7 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: override def metricsList(): List[String] = { List(config.totalEventsCount, config.dbReadCount, config.dbUpdateCount, config.failedEventCount, config.skippedEventCount, config.successEventCount, - config.cacheHitCount) + config.cacheHitCount, config.programCertIssueEventsCount) } override def processElement(event: Event, From 51d9d52f0c5d72bc780790febf5f3d9f1102021b Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Mon, 9 Oct 2023 16:05:21 +0530 Subject: [PATCH 58/88] Added the changes for program-pre-processor code changes for completedOn fix --- .../functions/ProgramCertPreProcessorFn.scala | 37 +++++++++++++------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index bcd935b7e..6c2e3a89a 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -17,7 +17,7 @@ import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} -import java.util.UUID +import java.util.{Date, UUID} import scala.collection.JavaConverters._ import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} import scala.collection.mutable @@ -73,14 +73,26 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: val leafNodeMap = mutable.Map[String, Int]() var isProgramCertificateToBeGenerated: Boolean = true; + var programCompletedOn: Date = null for (childNode <- contentDataForProgram) { val primaryCategory = childNode.get(config.primaryCategory).asInstanceOf[String] if (config.allowedPrimaryCategoryForProgram.contains(primaryCategory)) { val courseId: String = childNode.get(config.identifier).asInstanceOf[String] val userId: String = event.userId val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) - val isCertificateIssued = getCourseIssuedCertificateForUser(primaryFields)(metrics) + val courseEnrollment = getCourseEnrollment(primaryFields)(metrics) + val isCertificateIssued = courseEnrollment != null && !courseEnrollment.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) + var courseCompletedOn: Date = null; + if (isCertificateIssued) { + courseCompletedOn = courseEnrollment.getTimestamp("completedon") + if (programCompletedOn == null) { + programCompletedOn = courseCompletedOn + } else if (programCompletedOn.before(courseCompletedOn)) { + programCompletedOn = courseCompletedOn + } + } + breakable { if (!isCertificateIssued) { isProgramCertificateToBeGenerated = false; @@ -124,7 +136,7 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } else { isProgramCertificateToBeGenerated = false } - updateEnrolment(event.userId, batchId, courseParentId, programContentStatus, status, progressCount)(metrics) + updateEnrolment(event.userId, batchId, courseParentId, programContentStatus, status, progressCount, programCompletedOn)(metrics) } if (isProgramCertificateToBeGenerated) { @@ -157,9 +169,8 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: else new java.util.HashMap[String, AnyRef]() } - private def getCourseIssuedCertificateForUser(columns: Map[String, AnyRef])(implicit metrics: Metrics): Boolean = { + private def getCourseEnrollment(columns: Map[String, AnyRef])(implicit metrics: Metrics): Row = { logger.info("primary columns {}", columns) - var isCertificateIssued: Boolean = false; val selectWhere = QueryBuilder.select().all() .from(config.keyspace, config.userEnrolmentsTable). where() @@ -174,18 +185,16 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: logger.info("select query {}", selectWhere.toString) var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) if (null != row) { - row = row.filter(x => x.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])) != null).toList if (row.size() == 1) { - isCertificateIssued = true + row.asScala.get(0) } else { logger.error("More than one certificate" + columns) - isCertificateIssued = false + null } } else { logger.error("No Certificate Available" + columns) - isCertificateIssued = false + null } - isCertificateIssued } def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { @@ -209,16 +218,20 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } } - def updateEnrolment(userId: String, batchId: String, programId: String, contentStatus: java.util.Map[String, Integer], status: Int, progress: Int)(implicit metrics: Metrics): Unit = { + def updateEnrolment(userId: String, batchId: String, programId: String, contentStatus: java.util.Map[String, Integer], status: Int, progress: Int, programCompletedOn: Date)(implicit metrics: Metrics): Unit = { logger.info("Enrolment updated for userId: " + userId + " batchId: " + batchId) val updateQuery = QueryBuilder.update(config.keyspace, config.userEnrolmentsTable) .`with`(QueryBuilder.set("status", status)) .and(QueryBuilder.set("progress", progress)) .and(QueryBuilder.set("contentstatus", contentStatus)) .and(QueryBuilder.set("datetime", System.currentTimeMillis)) - .where(QueryBuilder.eq("userid", userId)) + if (status == 2) { + updateQuery.and(QueryBuilder.set("completedon", programCompletedOn)) + } + updateQuery.where(QueryBuilder.eq("userid", userId)) .and(QueryBuilder.eq("courseid", programId)) .and(QueryBuilder.eq("batchid", batchId)) + val result = cassandraUtil.upsert(updateQuery.toString) if (result) { metrics.incCounter(config.dbUpdateCount) From abd7c28a4ac4a3b26b68e03bc13fa72d5fab6a3a Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 10 Oct 2023 19:15:04 +0530 Subject: [PATCH 59/88] Added the changes for program-activity-aggregator code fix --- .../ProgramActivityAggregatesFunction.scala | 105 ++++++++++++------ 1 file changed, 69 insertions(+), 36 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 113813cc2..4da40fcfd 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -57,16 +57,23 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } override def process(key: Int, - context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, - events: Iterable[Map[String, AnyRef]], - metrics: Metrics): Unit = { + context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, + events: Iterable[Map[String, AnyRef]], + metrics: Metrics): Unit = { logger.debug("Input Events Size: " + events.toList.size) - val eventsInfo = events.filter(event => getProgramEvent(event)(metrics, config, httpUtil, cache) != null) - val inputUserConsumptionList: List[UserContentConsumption] = eventsInfo - .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) - .values.map(value => { + var updatedEventInfo: mutable.Buffer[Map[String, AnyRef]] = mutable.Buffer.empty + events.map(value => { + var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value)(metrics, config, httpUtil, cache) + if (eventInfoMap != null) { + updatedEventInfo ++= eventInfoMap + updatedEventInfo + } + }) + val inputUserConsumptionList: List[UserContentConsumption] = updatedEventInfo + .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) + .values.map(value => { metrics.incCounter(config.processedEnrolmentCount) val batchId = value.head(config.batchId).toString val userId = value.head(config.userId).toString @@ -76,7 +83,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) }).toList - if(inputUserConsumptionList.isEmpty) + if (inputUserConsumptionList.isEmpty) return // Fetch the content status from the table in batch format val dbUserConsumption: Map[String, UserContentConsumption] = getContentStatusFromDB(events.toList, metrics) @@ -216,15 +223,19 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val processedContents = inputData.contents.map { case (contentId, inputCC) => { // ContentStatus from DB. - val dbCC: ContentStatus = dbContents.getOrElse(contentId, ContentStatus(contentId, 0, 0, 0)) - val finalStatus = List(inputCC.status, dbCC.status).max // Final status is max of DB and Input ContentStatus. - val views = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { x.viewCount }) // View Count is sum of DB and Input ContentStatus. - val completion = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { x.completedCount }) // Completed Count is sum of DB and Input ContentStatus. - val eventsFor: List[String] = getEventActions(dbCC, inputCC) - // Merged ContentStatus. - (contentId, ContentStatus(contentId, finalStatus, completion, views, inputCC.fromInput, eventsFor)) - } + val dbCC: ContentStatus = dbContents.getOrElse(contentId, ContentStatus(contentId, 0, 0, 0)) + val finalStatus = List(inputCC.status, dbCC.status).max // Final status is max of DB and Input ContentStatus. + val views = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { + x.viewCount + }) // View Count is sum of DB and Input ContentStatus. + val completion = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { + x.completedCount + }) // Completed Count is sum of DB and Input ContentStatus. + val eventsFor: List[String] = getEventActions(dbCC, inputCC) + // Merged ContentStatus. + (contentId, ContentStatus(contentId, finalStatus, completion, views, inputCC.fromInput, eventsFor)) } + } val existingContents = processedContents.keys.toList val remainingContents = dbData.contents.filterKeys(key => !existingContents.contains(key)) @@ -329,8 +340,8 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC */ def getContentStatusFromEvent(contents: List[Map[String, AnyRef]]): Map[String, ContentStatus] = { val enrichedContents = contents.map(content => { - (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) - }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) + (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) + }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) .map(x => { val completedCount = if (x._2.intValue() == 2) 1 else 0 ContentStatus(x._1, x._2.intValue(), completedCount) @@ -418,16 +429,17 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } def getDBStatus(collectionId: String): String = { - val requestBody = s"""{ - | "request": { - | "filters": { - | "objectType": "Collection", - | "identifier": "$collectionId", - | "status": ["Live", "Unlisted", "Retired"] - | }, - | "fields": ["status"] - | } - |}""".stripMargin + val requestBody = + s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "$collectionId", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin val response = httpUtil.post(config.searchAPIURL, requestBody) if (response.status == 200) { @@ -477,7 +489,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) } else { logger.error("Failed to read content details for Id: " + identifier) - } + } logger.info("is program activity aggregator is skipping this event ? " + isValidProgram) isValidProgram } @@ -567,25 +579,46 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, cache: DataCache - ): Map[String, AnyRef] = { - var eventInfoMap: mutable.Map[String, AnyRef] = mutable.Map.empty[String, AnyRef] + ): mutable.Iterable[Map[String, AnyRef]] = { + var eventInfoMap: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] - if ((verifyPrimaryCategory(courseId)(metrics, config, httpUtil, cache))) { - eventInfoMap ++= eventData + val primaryCategory: String = eventData.getOrElse(config.primaryCategory, "").asInstanceOf[String] + val parentCollections: List[String] = eventData.getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + if (config.validProgramPrimaryCategory.contains(primaryCategory)) { + eventInfoMap += eventData + var eventInfo : Map[String, AnyRef] = Map.empty + eventInfo ++= eventData if (StringUtils.isEmpty(batchId)) { val row = getEnrolment(userId, courseId)(metrics) if (row != null) { - eventInfoMap += ("batchId" -> row.getString("batchid")) + eventInfo += ("batchId" -> row.getString("batchid")) } else { return null; } + eventInfoMap += eventInfo + } + } else if (StringUtils.isNotBlank(primaryCategory) && (primaryCategory.equals("Course") || primaryCategory.equals("Standalone Assessment")) + && !parentCollections.isEmpty) { + eventInfoMap += eventData + for (parentId <- parentCollections) { + val row = getEnrolment(userId, parentId)(metrics) + if (row != null) { + val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] + val eventInfoProgram = Map[String, AnyRef]("edata" -> + Map("contents" -> contentConsumption, + "userId" -> "${userId}", + "action" -> "batch-enrolment-update", + "iteration" -> 1, "batchId" -> row.getString("batchid"), + "courseId" -> parentId)) + eventInfoMap += eventInfoProgram + } } - eventInfoMap.toMap } else { - null; + logger.error("Not Valid Primary Category: " + primaryCategory + " parentCollections: " + parentCollections) } + eventInfoMap } def getEnrolment(userId: String, courseId: String)(implicit metrics: Metrics) = { From 1228b3e57e06153895a0927847621c36b9aec673 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 09:30:19 +0530 Subject: [PATCH 60/88] Added the changes for program-activity-aggregator for new edge case --- .../functions/ProgramActivityAggregatesFunction.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 4da40fcfd..f68226a3f 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -62,15 +62,17 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC metrics: Metrics): Unit = { logger.debug("Input Events Size: " + events.toList.size) - + logger.info("Input Event: " + events) var updatedEventInfo: mutable.Buffer[Map[String, AnyRef]] = mutable.Buffer.empty events.map(value => { var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value)(metrics, config, httpUtil, cache) + logger.info("EventInfoMap: " + eventInfoMap) if (eventInfoMap != null) { updatedEventInfo ++= eventInfoMap updatedEventInfo } }) + logger.info("Updated Event Info: "+ updatedEventInfo) val inputUserConsumptionList: List[UserContentConsumption] = updatedEventInfo .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { @@ -586,6 +588,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] val primaryCategory: String = eventData.getOrElse(config.primaryCategory, "").asInstanceOf[String] val parentCollections: List[String] = eventData.getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections ) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { eventInfoMap += eventData var eventInfo : Map[String, AnyRef] = Map.empty @@ -599,7 +602,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } eventInfoMap += eventInfo } - } else if (StringUtils.isNotBlank(primaryCategory) && (primaryCategory.equals("Course") || primaryCategory.equals("Standalone Assessment")) + } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) && !parentCollections.isEmpty) { eventInfoMap += eventData for (parentId <- parentCollections) { @@ -608,11 +611,12 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] val eventInfoProgram = Map[String, AnyRef]("edata" -> Map("contents" -> contentConsumption, - "userId" -> "${userId}", + "userId" -> userId, "action" -> "batch-enrolment-update", "iteration" -> 1, "batchId" -> row.getString("batchid"), "courseId" -> parentId)) eventInfoMap += eventInfoProgram + logger.info("EventMapInfoProgram:" + eventInfoProgram) } } } else { From 7e0f5935eb9b34bee597ed696e091d882f0413ed Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 09:46:48 +0530 Subject: [PATCH 61/88] Added the changes for program-activity-aggregator code fix config file change --- .../task/ProgramActivityAggregateUpdaterConfig.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala index 54afcfc9a..1fcf884e3 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -144,6 +144,7 @@ class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends val defaultHeaders = Map[String, String]("Content-Type" -> "application/json") val userAccBlockedErrCode = "UOS_USRRED0006" val name: String = "name" + val validProgramPrimaryCategory = List[String]("Program","Curated Program","Blended Program") } From cccccb1ef68400fb8a1643464f2fd9dd7f5849af Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 10:58:51 +0530 Subject: [PATCH 62/88] Added the changes for program-activity-aggregator type casting fix --- .../functions/ProgramActivityAggregatesFunction.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index f68226a3f..da39f1341 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -587,7 +587,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] val primaryCategory: String = eventData.getOrElse(config.primaryCategory, "").asInstanceOf[String] - val parentCollections: List[String] = eventData.getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val parentCollections: java.util.List[String] = eventData.getOrElse(config.parentCollections, new java.util.ArrayList[String]).asInstanceOf[java.util.List[String]] logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections ) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { eventInfoMap += eventData @@ -603,9 +603,9 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC eventInfoMap += eventInfo } } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) - && !parentCollections.isEmpty) { + && CollectionUtils.isNotEmpty(parentCollections)) { eventInfoMap += eventData - for (parentId <- parentCollections) { + for (parentId <- parentCollections.asScala.toList) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] From 6302c6ce14488d2cc7bf4641fac1a5a26914335c Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 12:12:51 +0530 Subject: [PATCH 63/88] Added the changes for program-activity-aggregator code typecasting --- .../functions/ProgramActivityAggregatesFunction.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index da39f1341..64d3b2527 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -609,12 +609,12 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val row = getEnrolment(userId, parentId)(metrics) if (row != null) { val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] - val eventInfoProgram = Map[String, AnyRef]("edata" -> - Map("contents" -> contentConsumption, + val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, "userId" -> userId, "action" -> "batch-enrolment-update", - "iteration" -> 1, "batchId" -> row.getString("batchid"), - "courseId" -> parentId)) + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) eventInfoMap += eventInfoProgram logger.info("EventMapInfoProgram:" + eventInfoProgram) } From 277595ac6c221aaaa016d30d4c1a7a064488c1ef Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 13:58:20 +0530 Subject: [PATCH 64/88] Added the changes for program-activity-aggregator code fix --- .../ProgramActivityAggregatesFunction.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 64d3b2527..0804b4d82 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -63,7 +63,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC logger.debug("Input Events Size: " + events.toList.size) logger.info("Input Event: " + events) - var updatedEventInfo: mutable.Buffer[Map[String, AnyRef]] = mutable.Buffer.empty + var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] events.map(value => { var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value)(metrics, config, httpUtil, cache) logger.info("EventInfoMap: " + eventInfoMap) @@ -72,7 +72,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC updatedEventInfo } }) - logger.info("Updated Event Info: "+ updatedEventInfo) + logger.info("Updated Event Info: " + updatedEventInfo) val inputUserConsumptionList: List[UserContentConsumption] = updatedEventInfo .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { @@ -84,7 +84,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val enrichedContents = getContentStatusFromEvent(userConsumedContents) UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) }).toList - + logger.info("the input user ConsumptionList:" + inputUserConsumptionList) if (inputUserConsumptionList.isEmpty) return // Fetch the content status from the table in batch format @@ -588,10 +588,10 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] val primaryCategory: String = eventData.getOrElse(config.primaryCategory, "").asInstanceOf[String] val parentCollections: java.util.List[String] = eventData.getOrElse(config.parentCollections, new java.util.ArrayList[String]).asInstanceOf[java.util.List[String]] - logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections ) + logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { eventInfoMap += eventData - var eventInfo : Map[String, AnyRef] = Map.empty + var eventInfo: Map[String, AnyRef] = Map.empty eventInfo ++= eventData if (StringUtils.isEmpty(batchId)) { val row = getEnrolment(userId, courseId)(metrics) @@ -610,11 +610,11 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC if (row != null) { val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, - "userId" -> userId, - "action" -> "batch-enrolment-update", - "iteration" -> 1.asInstanceOf[Integer], - "batchId" -> row.getString("batchid"), - "courseId" -> parentId) + "userId" -> userId, + "action" -> "batch-enrolment-update", + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) eventInfoMap += eventInfoProgram logger.info("EventMapInfoProgram:" + eventInfoProgram) } From 8fcfd734dc6af01290c199aa3504acda1f93a8b3 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 14:40:50 +0530 Subject: [PATCH 65/88] Added the changes for program-activity-aggregator event fix --- .../functions/ProgramActivityAggregatesFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 0804b4d82..ac06d705c 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -80,6 +80,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val batchId = value.head(config.batchId).toString val userId = value.head(config.userId).toString val courseId = value.head(config.courseId).toString + logger.info("courseId: " + courseId + " batchId: " + batchId) val userConsumedContents = value.head(config.contents).asInstanceOf[List[Map[String, AnyRef]]] val enrichedContents = getContentStatusFromEvent(userConsumedContents) UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) @@ -604,7 +605,6 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) && CollectionUtils.isNotEmpty(parentCollections)) { - eventInfoMap += eventData for (parentId <- parentCollections.asScala.toList) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { From 910d651423d4c0a42d789397a16fcc756d3b059e Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 16:17:00 +0530 Subject: [PATCH 66/88] Added the changes for program-activity-aggregator logger --- .../functions/ProgramActivityAggregatesFunction.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index ac06d705c..89fa4297f 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -91,6 +91,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC // Fetch the content status from the table in batch format val dbUserConsumption: Map[String, UserContentConsumption] = getContentStatusFromDB(events.toList, metrics) + logger.info("Content Consumption List :" + dbUserConsumption) // Final User's ContentConsumption after merging with DB data. // Here we have final viewcount, completedcount and identified the content which should generate AUDIT events for start and complete. val finalUserConsumptionList = inputUserConsumptionList.map(inputData => { @@ -98,10 +99,12 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC finalUserConsumption(inputData, dbData)(metrics) }) + logger.info("finalUserConsumptionList List :" + finalUserConsumptionList) // user_content_consumption update with viewcount and completedcout. val userConsumptionQueries = finalUserConsumptionList.flatMap(userConsumption => getContentConsumptionQueries(userConsumption)) updateDB(config.thresholdBatchWriteSize, userConsumptionQueries)(metrics) + logger.info("UserConsumption Queries List :" + userConsumptionQueries) val courseAggregations = finalUserConsumptionList.flatMap(userConsumption => { @@ -118,6 +121,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } else courseAggs }) + logger.info("courseAggregations Queries List :" + courseAggregations) // Saving all queries for course and it's children (only collection) aggregates. val aggQueries = courseAggregations.map(agg => getUserAggQuery(agg.activityAgg)) updateDB(config.thresholdBatchWriteSize, aggQueries)(metrics) @@ -128,9 +132,11 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val collectionProgressUpdateList = collectionProgressList.filter(progress => !progress.completed) context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) + logger.info("collectionProgressUpdateList Queries List :" + collectionProgressUpdateList) val collectionProgressCompleteList = collectionProgressList.filter(progress => progress.completed) context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) + logger.info("collectionProgressCompleteList Queries List :" + collectionProgressCompleteList) // Content AUDIT Event generation and pushing to output tag. finalUserConsumptionList.flatMap(userConsumption => contentAuditEvents(userConsumption)).foreach(event => context.output(config.auditEventOutputTag, gson.toJson(event))) } From 01b1aea3581dc32e153d39063420fdeb0dd143a1 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 12 Oct 2023 18:18:09 +0530 Subject: [PATCH 67/88] Added the changes for program-activity-aggregator Stream FIx --- .../ProgramActivityAggregateUpdaterStreamTask.scala | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index 9b3cc6652..4b10d7b9d 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -8,7 +8,7 @@ import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.streaming.api.scala._ import org.sunbird.job.programaggregate.domain.CollectionProgress import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.programaggregate.functions.{ProgramContentConsumptionDeDupFunction, ProgramActivityAggregatesFunction} +import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesFunction, ProgramContentConsumptionDeDupFunction, ProgramProgressCompleteFunction, ProgramProgressUpdateFunction} import org.sunbird.job.util.{FlinkUtil, HttpUtil} import java.io.File @@ -41,6 +41,16 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate progressStream.getSideOutput(config.failedEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaFailedEventTopic)) .name(config.programactivityAggFailedEventProducer).uid(config.programactivityAggFailedEventProducer) + progressStream.getSideOutput(config.collectionUpdateOutputTag).process(new ProgramProgressUpdateFunction(config)) + .name(config.collectionProgressUpdateFn).uid(config.collectionProgressUpdateFn).setParallelism(config.enrolmentCompleteParallelism) + val enrolmentCompleteStream = progressStream.getSideOutput(config.collectionCompleteOutputTag).process(new ProgramProgressCompleteFunction(config)) + .name(config.collectionCompleteFn).uid(config.collectionCompleteFn).setParallelism(config.enrolmentCompleteParallelism) + + /*enrolmentCompleteStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic)) + .name(config.certIssueEventProducer).uid(config.certIssueEventProducer) + enrolmentCompleteStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) + .name(config.enrolmentCompleteEventProducer).uid(config.enrolmentCompleteEventProducer)*/ + env.execute(config.jobName) } From 2016d410f5a725c23e7e882b6d71573b42f9583c Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 13 Oct 2023 11:16:26 +0530 Subject: [PATCH 68/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramActivityAggregatesFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 89fa4297f..b8e4b645e 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -614,7 +614,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC for (parentId <- parentCollections.asScala.toList) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { - val contentConsumption: List[String] = eventData.getOrElse(config.contents, List.empty[String]).asInstanceOf[List[String]] + val contentConsumption = eventData.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, "userId" -> userId, "action" -> "batch-enrolment-update", From fabc764ddf993fe333e009ac490587a334192a55 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Fri, 13 Oct 2023 16:08:42 +0530 Subject: [PATCH 69/88] Added the changes for program-activity-aggregator code changes for primaryCategory and parentCollections --- .../ProgramActivityAggregatesFunction.scala | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index b8e4b645e..898e67fd5 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -512,7 +512,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val courseMetadata = cache.getWithRetry(courseId) if (null == courseMetadata || courseMetadata.isEmpty) { val url = - config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory" + config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory,parentCollections" val response = getAPICall(url, "content")(config, httpUtil, metrics) val courseName = StringContext .processEscapes( @@ -524,11 +524,14 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC response.getOrElse(config.primaryCategory, "").asInstanceOf[String] ) .filter(_ >= ' ') + val parentCollections = response + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) courseInfoMap.put("courseName", courseName) courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) courseInfoMap } else { val courseName = StringContext @@ -543,11 +546,14 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC .asInstanceOf[String] ) .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() courseInfoMap.put("courseId", courseId) courseInfoMap.put("courseName", courseName) courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) courseInfoMap } @@ -593,8 +599,9 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] - val primaryCategory: String = eventData.getOrElse(config.primaryCategory, "").asInstanceOf[String] - val parentCollections: java.util.List[String] = eventData.getOrElse(config.parentCollections, new java.util.ArrayList[String]).asInstanceOf[java.util.List[String]] + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(courseId)(metrics, config, cache, httpUtil) + val primaryCategory: String = contentObj.get(config.primaryCategory).asInstanceOf[String] + val parentCollections: List[String] = contentObj.get(config.parentCollections).asInstanceOf[List[String]] logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { eventInfoMap += eventData @@ -610,8 +617,8 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC eventInfoMap += eventInfo } } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) - && CollectionUtils.isNotEmpty(parentCollections)) { - for (parentId <- parentCollections.asScala.toList) { + && !parentCollections.isEmpty) { + for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] From ab79a06a608e655d2589cf9a972b03f7ab74fac1 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 17 Oct 2023 14:19:35 +0530 Subject: [PATCH 70/88] Added the loggers for the activity aggregrator --- .../aggregate/functions/ActivityAggregatesFunction.scala | 8 ++++---- .../functions/CollectionProgressUpdateFunction.scala | 3 +++ .../functions/ProgramProgressUpdateFunction.scala | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index 41962a178..bfbc49385 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -85,7 +85,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti val userConsumptionQueries = finalUserConsumptionList.flatMap(userConsumption => getContentConsumptionQueries(userConsumption)) updateDB(config.thresholdBatchWriteSize, userConsumptionQueries)(metrics) - + logger.info("The value for userConsumptionQueries:" + userConsumptionQueries) val courseAggregations = finalUserConsumptionList.flatMap(userConsumption => { // Course Level Agg using the merged data of ContentConsumption per user, course and batch. @@ -107,13 +107,13 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti // Saving enrolment completion data. val collectionProgressList = courseAggregations.filter(agg => agg.collectionProgress.nonEmpty).map(agg => agg.collectionProgress.get) - + logger.info("The value for collectionProgressList:" + collectionProgressList) val collectionProgressUpdateList = collectionProgressList.filter(progress => !progress.completed) context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) - + logger.info("The value for collectionProgressUpdateList:" + collectionProgressUpdateList) val collectionProgressCompleteList = collectionProgressList.filter(progress => progress.completed) context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) - + logger.info("The value for collectionProgressCompleteList:" + collectionProgressCompleteList) // Content AUDIT Event generation and pushing to output tag. finalUserConsumptionList.flatMap(userConsumption => contentAuditEvents(userConsumption)).foreach(event => context.output(config.auditEventOutputTag, gson.toJson(event))) } diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala index 03ea1ddc0..9378368a7 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala @@ -39,8 +39,11 @@ class CollectionProgressUpdateFunction(config: ActivityAggregateUpdaterConfig)(i val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) (row != null && row.getInt("status") != 2) } else events + logger.info("The event at progress: "+ events) + logger.info("Pending Enrollment: " + pendingEnrolments) val enrolmentQueries = pendingEnrolments.map(collectionProgress => getEnrolmentUpdateQuery(collectionProgress)) updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + logger.info("enrolmentQueries Enrolement: " + enrolmentQueries) // Create and update the checksum to DeDup store for the input events. if (config.dedupEnabled) { events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala index 418c86473..677f2973e 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala @@ -39,7 +39,10 @@ class ProgramProgressUpdateFunction(config: ProgramActivityAggregateUpdaterConfi val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) (row != null && row.getInt("status") != 2) } else events + logger.info("The event at progress: "+ events) + logger.info("The pending Enrollment"+ pendingEnrolments) val enrolmentQueries = pendingEnrolments.map(collectionProgress => getEnrolmentUpdateQuery(collectionProgress)) + logger.info("The pending Enrollment"+ enrolmentQueries) updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) // Create and update the checksum to DeDup store for the input events. if (config.dedupEnabled) { From 363e9c2633ce6ae30cbd200f39caa3fb2384888d Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 17 Oct 2023 15:28:09 +0530 Subject: [PATCH 71/88] Added the changes for program-activity-aggregator code fix --- .../ProgramActivityAggregatesFunction.scala | 20 +- ...ogramContentConsumptionDeDupFunction.scala | 195 +++++++++++++++++- ...amActivityAggregateUpdaterStreamTask.scala | 2 +- 3 files changed, 190 insertions(+), 27 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala index 898e67fd5..ebc0c462f 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -60,20 +60,8 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, events: Iterable[Map[String, AnyRef]], metrics: Metrics): Unit = { - - logger.debug("Input Events Size: " + events.toList.size) - logger.info("Input Event: " + events) - var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] - events.map(value => { - var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value)(metrics, config, httpUtil, cache) - logger.info("EventInfoMap: " + eventInfoMap) - if (eventInfoMap != null) { - updatedEventInfo ++= eventInfoMap - updatedEventInfo - } - }) - logger.info("Updated Event Info: " + updatedEventInfo) - val inputUserConsumptionList: List[UserContentConsumption] = updatedEventInfo + logger.info("Event Info Inside ProgramActivityAggregrator: " + events) + val inputUserConsumptionList: List[UserContentConsumption] = events .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { metrics.incCounter(config.processedEnrolmentCount) @@ -474,7 +462,7 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC } else cacheStatus } - def verifyPrimaryCategory(identifier: String)( +/* def verifyPrimaryCategory(identifier: String)( metrics: Metrics, config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @@ -646,6 +634,6 @@ class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterC .and(QueryBuilder.eq("courseid", courseId)) metrics.incCounter(config.dbReadCount) cassandraUtil.findOne(selectWhere.toString) - } + }*/ } diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 8b08cda3b..9397f08b6 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -1,36 +1,54 @@ package org.sunbird.job.programaggregate.functions -import java.lang.reflect.Type -import java.security.MessageDigest -import java.util - +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} +import com.google.gson.Gson import com.google.gson.reflect.TypeToken +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration import org.apache.commons.lang3.StringUtils import org.apache.flink.api.common.typeinfo.TypeInformation import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory -import org.sunbird.job.cache.RedisConnect -import org.sunbird.job.programaggregate.common.DeDupHelper +import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.dedup.DeDupEngine -import org.sunbird.job.{BaseProcessFunction, Metrics} +import org.sunbird.job.programaggregate.common.DeDupHelper import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.{BaseProcessFunction, Metrics} +import java.lang.reflect.Type +import java.util +import java.util.concurrent.TimeUnit import scala.collection.JavaConverters._ +import scala.collection.mutable -class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { +class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType private[this] val logger = LoggerFactory.getLogger(classOf[ProgramContentConsumptionDeDupFunction]) var deDupEngine: DeDupEngine = _ + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() override def open(parameters: Configuration): Unit = { super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) deDupEngine.init() } override def close(): Unit = { + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } deDupEngine.close() super.close() } @@ -41,7 +59,18 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val isBatchEnrollmentEvent: Boolean = StringUtils.equalsIgnoreCase(eData.getOrElse(config.action, "").asInstanceOf[String], config.batchEnrolmentUpdateCode) if (isBatchEnrollmentEvent) { val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala - val filteredContents = contents.filter(x => x.get("status") == 2).toList + logger.info("Input Event: " + contents) + var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] + contents.map(value => { + var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value.asScala.toMap)(metrics, config, httpUtil, cache) + logger.info("EventInfoMap: " + eventInfoMap) + if (eventInfoMap != null) { + updatedEventInfo ++= eventInfoMap + updatedEventInfo + } + }) + logger.info("UpdatedEventInfoMap: " + updatedEventInfo) + val filteredContents = updatedEventInfo.filter(x => x.get("status") == 2).toList if (filteredContents.size == 0) metrics.incCounter(config.skipEventsCount) else @@ -53,7 +82,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd } override def metricsList(): List[String] = { - List(config.totalEventCount, config.skipEventsCount, config.batchEnrolmentUpdateEventCount) + List(config.totalEventCount, config.skipEventsCount, config.batchEnrolmentUpdateEventCount, config.dbReadCount) } def discardDuplicates(event: Map[String, AnyRef]): Boolean = { @@ -71,4 +100,150 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd } else false } else true } + + def getProgramEvent(eventData: Map[String, AnyRef])( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): mutable.Iterable[Map[String, AnyRef]] = { + var eventInfoMap: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] + val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] + val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(courseId)(metrics, config, cache, httpUtil) + val primaryCategory: String = contentObj.get(config.primaryCategory).asInstanceOf[String] + val parentCollections: List[String] = contentObj.get(config.parentCollections).asInstanceOf[List[String]] + logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) + if (config.validProgramPrimaryCategory.contains(primaryCategory)) { + eventInfoMap += eventData + var eventInfo: Map[String, AnyRef] = Map.empty + eventInfo ++= eventData + if (StringUtils.isEmpty(batchId)) { + val row = getEnrolment(userId, courseId)(metrics) + if (row != null) { + eventInfo += ("batchId" -> row.getString("batchid")) + } else { + return null; + } + eventInfoMap += eventInfo + } + } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) + && !parentCollections.isEmpty) { + for (parentId <- parentCollections) { + val row = getEnrolment(userId, parentId)(metrics) + if (row != null) { + val contentConsumption = eventData.getOrElse(config.contents, List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] + val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, + "userId" -> userId, + "action" -> "batch-enrolment-update", + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) + eventInfoMap += eventInfoProgram + logger.info("EventMapInfoProgram:" + eventInfoProgram) + } + } + } else { + logger.error("Not Valid Primary Category: " + primaryCategory + " parentCollections: " + parentCollections) + } + eventInfoMap + } + + def getEnrolment(userId: String, courseId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory,parentCollections" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + } diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index 4b10d7b9d..c62ef9581 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -26,7 +26,7 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.programActivityAggregateUpdaterConsumer) .uid(config.programActivityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) .rebalance - .process(new ProgramContentConsumptionDeDupFunction(config)).name(config.consumptionDeDupFn) + .process(new ProgramContentConsumptionDeDupFunction(config, httpUtil)).name(config.consumptionDeDupFn) .uid(config.consumptionDeDupFn).setParallelism(config.deDupProcessParallelism) .getSideOutput(config.uniqueConsumptionOutput) .keyBy(new ProgramActivityAggregatorKeySelector(config)) From bbcb63505c8c6bd72399c43b5086d2db455962ce Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 17 Oct 2023 17:51:29 +0530 Subject: [PATCH 72/88] Added the changes for program-activity-aggregator code fix --- ...ogramContentConsumptionDeDupFunction.scala | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 9397f08b6..e38e9728f 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -70,14 +70,8 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd } }) logger.info("UpdatedEventInfoMap: " + updatedEventInfo) - val filteredContents = updatedEventInfo.filter(x => x.get("status") == 2).toList - if (filteredContents.size == 0) - metrics.incCounter(config.skipEventsCount) - else - metrics.incCounter(config.batchEnrolmentUpdateEventCount) - filteredContents.map(c => { - (eData + ("contents" -> List(Map("contentId" -> c.get("contentId"), "status" -> c.get("status"))))).toMap - }).filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d)) + + updatedEventInfo.filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d)) } else metrics.incCounter(config.skipEventsCount) } @@ -117,24 +111,14 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { eventInfoMap += eventData - var eventInfo: Map[String, AnyRef] = Map.empty - eventInfo ++= eventData - if (StringUtils.isEmpty(batchId)) { - val row = getEnrolment(userId, courseId)(metrics) - if (row != null) { - eventInfo += ("batchId" -> row.getString("batchid")) - } else { - return null; - } - eventInfoMap += eventInfo - } } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) && !parentCollections.isEmpty) { for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] - val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, + val filteredContents = contentConsumption.filter(x => x.get("status") == 2) + val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, "userId" -> userId, "action" -> "batch-enrolment-update", "iteration" -> 1.asInstanceOf[Integer], From 2f1a0f5be282aa29766efe136dd517497b33c923 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 17 Oct 2023 18:30:24 +0530 Subject: [PATCH 73/88] Added the changes for program-activity-aggregator DeDup fix --- .../ProgramContentConsumptionDeDupFunction.scala | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index e38e9728f..bafc6385d 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -61,14 +61,12 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala logger.info("Input Event: " + contents) var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] - contents.map(value => { - var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(value.asScala.toMap)(metrics, config, httpUtil, cache) - logger.info("EventInfoMap: " + eventInfoMap) - if (eventInfoMap != null) { - updatedEventInfo ++= eventInfoMap - updatedEventInfo - } - }) + var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(event.asScala.toMap)(metrics, config, httpUtil, cache) + logger.info("EventInfoMap: " + eventInfoMap) + if (eventInfoMap != null) { + updatedEventInfo ++= eventInfoMap + } + logger.info("UpdatedEventInfoMap: " + updatedEventInfo) updatedEventInfo.filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d)) From 353b12a35282cdd284c8b430c2fbd6e98740245c Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Tue, 17 Oct 2023 19:17:17 +0530 Subject: [PATCH 74/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index bafc6385d..7194cbe71 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -61,7 +61,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala logger.info("Input Event: " + contents) var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] - var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(event.asScala.toMap)(metrics, config, httpUtil, cache) + var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(eData.toMap)(metrics, config, httpUtil, cache) logger.info("EventInfoMap: " + eventInfoMap) if (eventInfoMap != null) { updatedEventInfo ++= eventInfoMap @@ -99,6 +99,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd httpUtil: HttpUtil, cache: DataCache ): mutable.Iterable[Map[String, AnyRef]] = { + logger.info("EventInfo" + eventData) var eventInfoMap: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] From f359fc9f906b6735fe1960b0a2a186cf49075a5e Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 09:20:49 +0530 Subject: [PATCH 75/88] Added the changes for program-activity-aggregator code fix typeCasting --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 7194cbe71..65882a650 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -115,7 +115,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { - val contentConsumption = eventData.getOrElse(config.contents, List[Map[String, AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala val filteredContents = contentConsumption.filter(x => x.get("status") == 2) val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, "userId" -> userId, From ccc6bd57b10df7b69ae28eb0babb5fd05385279b Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 09:51:13 +0530 Subject: [PATCH 76/88] Added the changes for program-activity-aggregator code fix typecasting --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 65882a650..4df2295c1 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -117,7 +117,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala val filteredContents = contentConsumption.filter(x => x.get("status") == 2) - val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, + val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents.toList, "userId" -> userId, "action" -> "batch-enrolment-update", "iteration" -> 1.asInstanceOf[Integer], From c00ea2af2f1932c9fbd100cdb5befb910b837edb Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 11:06:02 +0530 Subject: [PATCH 77/88] Added the changes for program-activity-aggregator typecasting --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 4df2295c1..609021421 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -115,7 +115,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { - val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap) val filteredContents = contentConsumption.filter(x => x.get("status") == 2) val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents.toList, "userId" -> userId, From 7e2584a8a608a049827c273d6a617f59b4e301d6 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 11:51:11 +0530 Subject: [PATCH 78/88] Added the changes for program-activity-aggregator code fix --- ...ogramContentConsumptionDeDupFunction.scala | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 609021421..48067ca0b 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -63,7 +63,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(eData.toMap)(metrics, config, httpUtil, cache) logger.info("EventInfoMap: " + eventInfoMap) - if (eventInfoMap != null) { + if (eventInfoMap.nonEmpty) { updatedEventInfo ++= eventInfoMap } @@ -117,14 +117,16 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap) val filteredContents = contentConsumption.filter(x => x.get("status") == 2) - val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents.toList, - "userId" -> userId, - "action" -> "batch-enrolment-update", - "iteration" -> 1.asInstanceOf[Integer], - "batchId" -> row.getString("batchid"), - "courseId" -> parentId) - eventInfoMap += eventInfoProgram - logger.info("EventMapInfoProgram:" + eventInfoProgram) + if(filteredContents.nonEmpty) { + val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, + "userId" -> userId, + "action" -> "batch-enrolment-update", + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) + eventInfoMap += eventInfoProgram + logger.info("EventMapInfoProgram:" + eventInfoProgram) + } } } } else { From 05caa3cd461021dad6cfafae1001ad8cc6267ecd Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 12:21:31 +0530 Subject: [PATCH 79/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 48067ca0b..c554a22ca 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -82,6 +82,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val userId = event.getOrElse(config.userId, "").asInstanceOf[String] val courseId = event.getOrElse(config.courseId, "").asInstanceOf[String] val batchId = event.getOrElse(config.batchId, "").asInstanceOf[String] + logger.info("Event List inside discardDuplicates" + event) val contents = event.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] if (contents.nonEmpty) { val content = contents.head @@ -115,7 +116,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) if (row != null) { - val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap) + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList val filteredContents = contentConsumption.filter(x => x.get("status") == 2) if(filteredContents.nonEmpty) { val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, From 23e7a7c368da9de6720d82c65749f715854144c2 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 12:44:11 +0530 Subject: [PATCH 80/88] Added the changes for program-activity-aggregator code fix type casting --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index c554a22ca..0be8ceab9 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -110,7 +110,10 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val parentCollections: List[String] = contentObj.get(config.parentCollections).asInstanceOf[List[String]] logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) if (config.validProgramPrimaryCategory.contains(primaryCategory)) { - eventInfoMap += eventData + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList + val mergedMap = eventData.updated(config.contents, contentConsumption) + eventInfoMap += mergedMap + logger.info("Inside Valid Primary " + mergedMap) } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) && !parentCollections.isEmpty) { for (parentId <- parentCollections) { From f803cfb4832cd89aea73f63dacbd100aa79e6145 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 13:19:22 +0530 Subject: [PATCH 81/88] Added the logging --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 0be8ceab9..b83c8934f 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -118,9 +118,12 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd && !parentCollections.isEmpty) { for (parentId <- parentCollections) { val row = getEnrolment(userId, parentId)(metrics) + logger.info("Enrollment: " + row) if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList + logger.info("contentConsumption: " + contentConsumption) val filteredContents = contentConsumption.filter(x => x.get("status") == 2) + logger.info("filteredContents: " + filteredContents) if(filteredContents.nonEmpty) { val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, "userId" -> userId, From b2dc98ae3af6ded0bd45057ddc81ac3e87dc5900 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 13:56:36 +0530 Subject: [PATCH 82/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index b83c8934f..31babfe43 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -122,7 +122,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList logger.info("contentConsumption: " + contentConsumption) - val filteredContents = contentConsumption.filter(x => x.get("status") == 2) + val filteredContents = contentConsumption.filter(x => x.get("status").asInstanceOf[Integer] == 2) logger.info("filteredContents: " + filteredContents) if(filteredContents.nonEmpty) { val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, From 96c5d3ebccd8ec71587be03c8dbbee8c9914d175 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 14:06:49 +0530 Subject: [PATCH 83/88] Added the changes for program-activity-aggregator code typecasting --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 31babfe43..726464608 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -122,7 +122,7 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd if (row != null) { val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList logger.info("contentConsumption: " + contentConsumption) - val filteredContents = contentConsumption.filter(x => x.get("status").asInstanceOf[Integer] == 2) + val filteredContents = contentConsumption.filter(x => x.get("status").asInstanceOf[Double].toInt == 2) logger.info("filteredContents: " + filteredContents) if(filteredContents.nonEmpty) { val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, From 855c5ad5e9811dc143da334bc2db836c0d3ef227 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Wed, 18 Oct 2023 14:54:37 +0530 Subject: [PATCH 84/88] Added the changes for program-activity-aggregator code fix --- .../functions/ProgramContentConsumptionDeDupFunction.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala index 726464608..c09d32901 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -120,9 +120,9 @@ class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpd val row = getEnrolment(userId, parentId)(metrics) logger.info("Enrollment: " + row) if (row != null) { - val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala logger.info("contentConsumption: " + contentConsumption) - val filteredContents = contentConsumption.filter(x => x.get("status").asInstanceOf[Double].toInt == 2) + val filteredContents = contentConsumption.filter(x => x.get("status") == 2).map(_.asScala.toMap).toList logger.info("filteredContents: " + filteredContents) if(filteredContents.nonEmpty) { val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, From ffe40228bb59de2f65d4c3a3c117553fd48c233d Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Thu, 19 Oct 2023 13:09:30 +0530 Subject: [PATCH 85/88] Optimized code to reduce cassandra queries --- .../functions/ProgramCertPreProcessorFn.scala | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 6c2e3a89a..8d326b62c 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -62,13 +62,14 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: try { val getParentIdForCourse = event.parentCollections if (!getParentIdForCourse.isEmpty) { + val enrolmentRecords = getAllEnrolments(event.userId)(metrics) for (courseParentId <- getParentIdForCourse) { - val programEnrollmentStatus = getEnrolment(event.userId, courseParentId)(metrics) + val programEnrollmentRow = getEnrollmentRecord(enrolmentRecords, courseParentId) //if enrolled into program - if (null != programEnrollmentStatus && programEnrollmentStatus.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty) { + if (programEnrollmentRow.isDefined && programEnrollmentRow.get.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty) { val programHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) if (!programHierarchy.isEmpty) { - val batchId: String = programEnrollmentStatus.getString(config.dbBatchId) + val batchId: String = programEnrollmentRow.get.getString(config.dbBatchId) val contentDataForProgram = programHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] val leafNodeMap = mutable.Map[String, Int]() @@ -79,13 +80,12 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: if (config.allowedPrimaryCategoryForProgram.contains(primaryCategory)) { val courseId: String = childNode.get(config.identifier).asInstanceOf[String] val userId: String = event.userId - val primaryFields = Map(config.dbUserId.toLowerCase() -> userId, config.dbCourseId.toLowerCase -> courseId) - val courseEnrollment = getCourseEnrollment(primaryFields)(metrics) - val isCertificateIssued = courseEnrollment != null && !courseEnrollment.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty + val courseEnrollmentRow = getEnrollmentRecord(enrolmentRecords, courseId) + val isCertificateIssued = courseEnrollmentRow.isDefined && !courseEnrollmentRow.get.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) var courseCompletedOn: Date = null; if (isCertificateIssued) { - courseCompletedOn = courseEnrollment.getTimestamp("completedon") + courseCompletedOn = courseEnrollmentRow.get.getTimestamp("completedon") if (programCompletedOn == null) { programCompletedOn = courseCompletedOn } else if (programCompletedOn.before(courseCompletedOn)) { @@ -107,9 +107,9 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } } if (!leafNodeMap.isEmpty) { - val programContentStatus = Option(programEnrollmentStatus.getMap( + val programContentStatus = Option(programEnrollmentRow.get.getMap( config.contentStatus, TypeToken.of(classOf[String]), TypeToken.of(classOf[Integer]))).head - var progressCount: Integer = Option(programEnrollmentStatus.getInt(config.progress)).head + var progressCount: Integer = Option(programEnrollmentRow.get.getInt(config.progress)).head var updateCount = 0 @@ -250,4 +250,24 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: context.output(config.generateCertificateOutputTag, event) metrics.incCounter(config.programCertIssueEventsCount) } + + def getAllEnrolments(userId: String)(implicit metrics: Metrics): java.util.List[Row] = { + val selectWhere: Select.Where = QueryBuilder.select(config.dbUserId, config.dbCourseId, config.dbBatchId, config.contentStatus, config.progress, config.issuedCertificates, "completedon") + .from(config.keyspace, config.userEnrolmentsTable).where() + selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.find(selectWhere.toString) + } + + def getEnrollmentRecord(enrollList: java.util.List[Row], courseId: String): Option[Row] = { + if(null != enrollList) { + enrollList.asScala.find { row => + val courseid = row.getString("courseid") + val active = row.getBool("active") + courseid == courseId && active + } + } else { + None + } + } } From 661f1423caf5f92771b36e3c3205aba05ad04fe5 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 19 Oct 2023 15:44:30 +0530 Subject: [PATCH 86/88] Added the changes for new function for Activity Aggregrator --- ...ctivityAggregatesEnrolUpdateFunction.scala | 268 ++++++++++++++++++ ...amActivityAggregateUpdaterStreamTask.scala | 8 +- 2 files changed, 272 insertions(+), 4 deletions(-) create mode 100644 program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala new file mode 100644 index 000000000..ea720f482 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala @@ -0,0 +1,268 @@ +package org.sunbird.job.programaggregate.functions + +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} +import com.google.common.reflect.TypeToken +import com.google.gson.Gson +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.{Metrics, WindowBaseProcessFunction} + +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.`seq AsJavaList` +import scala.collection.mutable.ListBuffer + + + +class ProgramActivityAggregatesEnrolUpdateFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) + (implicit val stringTypeInfo: TypeInformation[String]) + extends WindowBaseProcessFunction[Map[String, AnyRef], String, Int](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramActivityAggregatesEnrolUpdateFunction]) + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() + + override def metricsList(): List[String] = { + List(config.failedEventCount, config.dbUpdateCount, config.dbReadCount, config.cacheHitCount, config.cacheMissCount, config.processedEnrolmentCount, config.retiredCCEventsCount) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) + } + + override def close(): Unit = { + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } + super.close() + } + + override def process(key: Int, + context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, + events: Iterable[Map[String, AnyRef]], + metrics: Metrics): Unit = { + logger.info("Event Info Inside ProgramActivityAggregrator: " + events) + val inputUserConsumptionList: List[UserContentConsumption] = events + .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) + .values.map(value => { + metrics.incCounter(config.processedEnrolmentCount) + val batchId = value.head(config.batchId).toString + val userId = value.head(config.userId).toString + val courseId = value.head(config.courseId).toString + logger.info("courseId: " + courseId + " batchId: " + batchId) + val userConsumedContents = value.head(config.contents).asInstanceOf[List[Map[String, AnyRef]]] + val enrichedContents = getContentStatusFromEvent(userConsumedContents) + UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) + }).toList + logger.info("the input user ConsumptionList:" + inputUserConsumptionList) + if (inputUserConsumptionList.isEmpty) + return + + val updateProgramEnrollments = updateProgramEnrollment(inputUserConsumptionList)(metrics) + + val collectionProgressUpdateList = updateProgramEnrollments.filter(progress => !progress.completed) + context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) + + logger.info("collectionProgressUpdateList Queries List :" + collectionProgressUpdateList) + val collectionProgressCompleteList = updateProgramEnrollments.filter(progress => progress.completed) + context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) + + logger.info("collectionProgressCompleteList Queries List :" + collectionProgressCompleteList) + } + + def getContentStatusFromEvent(contents: List[Map[String, AnyRef]]): Map[String, ContentStatus] = { + val enrichedContents = contents.map(content => { + (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) + }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) + .map(x => { + val completedCount = if (x._2.intValue() == 2) 1 else 0 + ContentStatus(x._1, x._2.intValue(), completedCount) + }).groupBy(f => f.contentId) + + enrichedContents.map(content => { + val consumedList = content._2 + val finalStatus = consumedList.map(x => x.status).max + val views = sumFunc(consumedList, (x: ContentStatus) => { + x.viewCount + }) + val completion = sumFunc(consumedList, (x: ContentStatus) => { + x.completedCount + }) + (content._1, ContentStatus(content._1, finalStatus, completion, views)) + }) + } + + /** + * Computation of Sum for viewCount and completedCount. + */ + private def sumFunc(list: List[ContentStatus], valFunc: ContentStatus => Int): Int = list.map(x => valFunc(x)).sum + + def updateProgramEnrollment(events: List[UserContentConsumption])(implicit metrics: Metrics): List[CollectionProgress] = { + + val contentEnrollProgress: List[CollectionProgress] = events.flatMap { collectionProgress => + val updatedEnrolContentConsumption = updateEnrolContentConsumption(collectionProgress)(metrics) + if (updatedEnrolContentConsumption != null) + programEnrolConsumption(updatedEnrolContentConsumption)(metrics) + else + None + } + contentEnrollProgress + } + + def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq(config.userId, userId)) + .and(QueryBuilder.eq(config.courseId, programId)) + metrics.incCounter(config.dbReadCount) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + if (row.size() == 1) { + row.asScala.get(0) + } else { + logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) + null + } + } else { + logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) + null + } + } + + def readFromCache(key: String, metrics: Metrics): List[String] = { + metrics.incCounter(config.cacheHitCount) + val list = cache.getKeyMembers(key) + if (CollectionUtils.isEmpty(list)) { + metrics.incCounter(config.cacheMissCount) + logger.info("Redis cache (smembers) not available for key: " + key) + } + list.asScala.toList + } + + def updateEnrolContentConsumption(userConsumption: UserContentConsumption)(implicit metrics: Metrics): UserContentConsumption = { + val programEnrollmentStatus = getEnrolment(userConsumption.userId, userConsumption.courseId)(metrics) + if (programEnrollmentStatus != null && programEnrollmentStatus.getInt("status") != 2) { + val programContentStatusList = ListBuffer[Map[String, AnyRef]]() + val programContentStatus = Option(programEnrollmentStatus.getMap( + config.contentStatus, TypeToken.of(classOf[String]), TypeToken.of(classOf[Integer]))).head + for ((key, value) <- userConsumption.contents) { + // Check if the key is present in leafNodeMap + if (programContentStatus.get(key) != null) { + if (value.status == 2 && programContentStatus.get(key) != 2) { + // Update progress in contentStatus for the matching key + programContentStatus.put(key, value.status) + } + } else { + programContentStatus.put(key, value.status) + } + } + + for( (key, value) <- programContentStatus.asScala) { + val contentStatusMap = Map( + config.contentId -> key, + config.status -> value + ) + programContentStatusList += contentStatusMap + } + // Add programContentStatusMap to programContentStatusList + val updatedContent = getContentStatusFromEvent(programContentStatusList.toList) + val updatedUserConsumption = userConsumption.copy(contents = updatedContent) + updatedUserConsumption + } else { + null + } + } + + def programEnrolConsumption(userConsumption: UserContentConsumption)(implicit metrics: Metrics): Option[CollectionProgress] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + val key = s"$courseId:$courseId:${config.leafNodes}" + val leafNodes = readFromCache(key, metrics).distinct + if (leafNodes.isEmpty) { + logger.error(s"leaf nodes are not available for: $key") + //context.output(config.failedEventOutputTag, gson.toJson(userConsumption)) + val status = getCollectionStatus(courseId) + if (StringUtils.equals("Retired", status)) { + metrics.incCounter(config.retiredCCEventsCount) + println(s"contents consumed from a retired collection: $courseId") + logger.warn(s"contents consumed from a retired collection: $courseId") + None + } else { + metrics.incCounter(config.failedEventCount) + val message = s"leaf nodes are not available for a published collection: $courseId" + logger.error(message) + throw new Exception(message) + } + } else { + val completedCount = leafNodes.intersect(userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct).size + val contentStatus = userConsumption.contents.map(cc => (cc._2.contentId, cc._2.status)).toMap + val inputContents = userConsumption.contents.filter(cc => cc._2.fromInput).keys.toList + val collectionProgress = if (completedCount >= leafNodes.size) { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, new java.util.Date(), contentStatus, inputContents, true)) + } else { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, null, contentStatus, inputContents)) + } + return collectionProgress + } + } + + def getCollectionStatus(collectionId: String): String = { + val cacheStatus = collectionStatusCache.getNonExpired(collectionId).getOrElse("") + if (StringUtils.isEmpty(cacheStatus)) { + val dbStatus = getDBStatus(collectionId) + collectionStatusCache = collectionStatusCache.putClocked(collectionId, dbStatus)._2 + dbStatus + } else cacheStatus + } + + def getDBStatus(collectionId: String): String = { + val requestBody = + s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "$collectionId", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + val response = httpUtil.post(config.searchAPIURL, requestBody) + if (response.status == 200) { + val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) + val result = responseBody.getOrDefault("result", new java.util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] + val count = result.getOrDefault("count", 0.asInstanceOf[Number]).asInstanceOf[Number].intValue() + if (count > 0) { + val list = result.getOrDefault("content", new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + list.asScala.head.get("status").asInstanceOf[String] + } else throw new Exception(s"There are no published or retired collection with id: $collectionId") + } else { + logger.error("search-service error: " + response.body) + throw new Exception("search-service not returning error:" + response.status) + } + } +} + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala index c62ef9581..eac0a2783 100644 --- a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -8,7 +8,7 @@ import org.apache.flink.api.java.utils.ParameterTool import org.apache.flink.streaming.api.scala._ import org.sunbird.job.programaggregate.domain.CollectionProgress import org.sunbird.job.connector.FlinkKafkaConnector -import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesFunction, ProgramContentConsumptionDeDupFunction, ProgramProgressCompleteFunction, ProgramProgressUpdateFunction} +import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesEnrolUpdateFunction, ProgramActivityAggregatesFunction, ProgramContentConsumptionDeDupFunction, ProgramProgressCompleteFunction, ProgramProgressUpdateFunction} import org.sunbird.job.util.{FlinkUtil, HttpUtil} import java.io.File @@ -31,7 +31,7 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate .getSideOutput(config.uniqueConsumptionOutput) .keyBy(new ProgramActivityAggregatorKeySelector(config)) .countWindow(config.thresholdBatchReadSize) - .process(new ProgramActivityAggregatesFunction(config, httpUtil)) + .process(new ProgramActivityAggregatesEnrolUpdateFunction(config, httpUtil)) .name(config.programactivityAggregateUpdaterFn) .uid(config.programactivityAggregateUpdaterFn) .setParallelism(config.activityAggregateUpdaterParallelism) @@ -46,9 +46,9 @@ class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregate val enrolmentCompleteStream = progressStream.getSideOutput(config.collectionCompleteOutputTag).process(new ProgramProgressCompleteFunction(config)) .name(config.collectionCompleteFn).uid(config.collectionCompleteFn).setParallelism(config.enrolmentCompleteParallelism) - /*enrolmentCompleteStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic)) + enrolmentCompleteStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic)) .name(config.certIssueEventProducer).uid(config.certIssueEventProducer) - enrolmentCompleteStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) + /*enrolmentCompleteStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) .name(config.enrolmentCompleteEventProducer).uid(config.enrolmentCompleteEventProducer)*/ env.execute(config.jobName) From a0353e6285a8b13200499dc98a9ee234480173b9 Mon Sep 17 00:00:00 2001 From: sahilchaudhary Date: Thu, 19 Oct 2023 17:01:02 +0530 Subject: [PATCH 87/88] Added the changes for program-cert-pre-processor code fix --- .../job/programcert/functions/ProgramCertPreProcessorFn.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala index 8d326b62c..753fe2516 100644 --- a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -252,7 +252,7 @@ class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: } def getAllEnrolments(userId: String)(implicit metrics: Metrics): java.util.List[Row] = { - val selectWhere: Select.Where = QueryBuilder.select(config.dbUserId, config.dbCourseId, config.dbBatchId, config.contentStatus, config.progress, config.issuedCertificates, "completedon") + val selectWhere: Select.Where = QueryBuilder.select(config.dbUserId, config.dbCourseId, config.dbBatchId, config.contentStatus, config.progress, config.issuedCertificates, "completedon", "active") .from(config.keyspace, config.userEnrolmentsTable).where() selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) metrics.incCounter(config.dbReadCount) From 3a43796d710ec054ae8addd84132937815ca8df2 Mon Sep 17 00:00:00 2001 From: karthik-tarento Date: Tue, 24 Oct 2023 11:45:52 +0530 Subject: [PATCH 88/88] Disabled user feed notification --- .../job/certgen/functions/CertificateGeneratorFunction.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index 893a98c3c..97668b61f 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -309,7 +309,7 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: logger.info("pushAuditEvent: certificate audit event success {}", audit) context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) - context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) + //context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { metrics.incCounter(config.failedEventCount) throw new Exception(s"Update certificates to enrolments failed: ${event}")