diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala index d80e12e7d..bfbc49385 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/ActivityAggregatesFunction.scala @@ -2,7 +2,6 @@ package org.sunbird.job.aggregate.functions import java.lang.reflect.Type import java.util.concurrent.TimeUnit - import com.datastax.driver.core.Row import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} import com.google.gson.Gson @@ -19,10 +18,11 @@ import org.slf4j.LoggerFactory import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.aggregate.domain.{UserContentConsumption, _} import org.sunbird.job.aggregate.task.ActivityAggregateUpdaterConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, ScalaJsonUtil} import org.sunbird.job.{Metrics, WindowBaseProcessFunction} import scala.collection.JavaConverters._ +import scala.language.postfixOps class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) (implicit val stringTypeInfo: TypeInformation[String]) @@ -57,8 +57,9 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti events: Iterable[Map[String, AnyRef]], metrics: Metrics): Unit = { - logger.debug("Input Events Size: " + events.toList.size) + logger.info("Input Events : " + JSONUtil.serialize(events.toList)) val inputUserConsumptionList: List[UserContentConsumption] = events + .filter(event=> verifyPrimaryCategory(event.getOrElse(config.courseId, "").asInstanceOf[String])(metrics, config, httpUtil, cache)) .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) .values.map(value => { metrics.incCounter(config.processedEnrolmentCount) @@ -84,7 +85,7 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti val userConsumptionQueries = finalUserConsumptionList.flatMap(userConsumption => getContentConsumptionQueries(userConsumption)) updateDB(config.thresholdBatchWriteSize, userConsumptionQueries)(metrics) - + logger.info("The value for userConsumptionQueries:" + userConsumptionQueries) val courseAggregations = finalUserConsumptionList.flatMap(userConsumption => { // Course Level Agg using the merged data of ContentConsumption per user, course and batch. @@ -106,13 +107,13 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti // Saving enrolment completion data. val collectionProgressList = courseAggregations.filter(agg => agg.collectionProgress.nonEmpty).map(agg => agg.collectionProgress.get) - + logger.info("The value for collectionProgressList:" + collectionProgressList) val collectionProgressUpdateList = collectionProgressList.filter(progress => !progress.completed) context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) - + logger.info("The value for collectionProgressUpdateList:" + collectionProgressUpdateList) val collectionProgressCompleteList = collectionProgressList.filter(progress => progress.completed) context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) - + logger.info("The value for collectionProgressCompleteList:" + collectionProgressCompleteList) // Content AUDIT Event generation and pushing to output tag. finalUserConsumptionList.flatMap(userConsumption => contentAuditEvents(userConsumption)).foreach(event => context.output(config.auditEventOutputTag, gson.toJson(event))) } @@ -444,5 +445,133 @@ class ActivityAggregatesFunction(config: ActivityAggregateUpdaterConfig, httpUti dbStatus } else cacheStatus } -} + def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: ActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (!contentObj.isEmpty) { + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory != "Program" + || primaryCategory != "Curated Program" + || primaryCategory != "Blended Program")) { + isValidProgram = true + } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) + } + logger.info("is activity aggregator is skipping this event ? " + isValidProgram) + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + "/" + courseId + "?fields=identifier,name,versionKey,parentCollections,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + response.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + courseMetadata.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse("parentCollections", List.empty[String]) + .asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("versionKey", versionKey) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } +} diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala index 03ea1ddc0..9378368a7 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/functions/CollectionProgressUpdateFunction.scala @@ -39,8 +39,11 @@ class CollectionProgressUpdateFunction(config: ActivityAggregateUpdaterConfig)(i val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) (row != null && row.getInt("status") != 2) } else events + logger.info("The event at progress: "+ events) + logger.info("Pending Enrollment: " + pendingEnrolments) val enrolmentQueries = pendingEnrolments.map(collectionProgress => getEnrolmentUpdateQuery(collectionProgress)) updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + logger.info("enrolmentQueries Enrolement: " + enrolmentQueries) // Create and update the checksum to DeDup store for the input events. if (config.dedupEnabled) { events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) diff --git a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala index f36edaf4c..1e024f408 100644 --- a/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala +++ b/activity-aggregate-updater/src/main/scala/org/sunbird/job/aggregate/task/ActivityAggregateUpdaterConfig.scala @@ -132,4 +132,13 @@ class ActivityAggregateUpdaterConfig(override val config: Config) extends BaseJo val searchServiceBasePath: String = config.getString("service.search.basePath") val searchAPIURL = searchServiceBasePath + "/v3/search" + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase + "/content/v3/read/" + val name: String = "name" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val skippedEventCount = "skipped-events-count" + } diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala index 3f946a1b5..1e9452eef 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/functions/IssueCertificateHelper.scala @@ -3,33 +3,37 @@ package org.sunbird.job.collectioncert.functions import java.text.SimpleDateFormat import com.datastax.driver.core.querybuilder.QueryBuilder import com.datastax.driver.core.{Row, TypeTokens} +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang3.StringUtils +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper import org.slf4j.LoggerFactory import org.sunbird.job.Metrics import org.sunbird.job.cache.DataCache import org.sunbird.job.collectioncert.domain.{AssessedUser, AssessmentUserAttempt, BEJobRequestEvent, EnrolledUser, Event, EventObject} import org.sunbird.job.collectioncert.task.CollectionCertPreProcessorConfig -import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil, JSONUtil} import scala.collection.JavaConverters._ trait IssueCertificateHelper { private[this] val logger = LoggerFactory.getLogger(classOf[CollectionCertPreProcessorFn]) - def issueCertificate(event:Event, template: Map[String, String])(cassandraUtil: CassandraUtil, cache:DataCache, contentCache: DataCache, metrics: Metrics, config: CollectionCertPreProcessorConfig, httpUtil: HttpUtil): String = { //validCriteria logger.info("issueCertificate i/p event =>"+event) val criteria = validateTemplate(template, event.batchId)(config) //validateEnrolmentCriteria val certName = template.getOrElse(config.name, "") + logger.info("CertName" + certName) val additionalProps: Map[String, List[String]] = ScalaJsonUtil.deserialize[Map[String, List[String]]](template.getOrElse("additionalProps", "{}")) val enrolledUser: EnrolledUser = validateEnrolmentCriteria(event, criteria.getOrElse(config.enrollment, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], certName, additionalProps)(metrics, cassandraUtil, config) + logger.info("enrolledUser" + enrolledUser) //validateAssessmentCriteria val assessedUser = validateAssessmentCriteria(event, criteria.getOrElse(config.assessment, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], enrolledUser.userId, additionalProps)(metrics, cassandraUtil, contentCache, config) + logger.info("assessedUser" + assessedUser) //validateUserCriteria val userDetails = validateUser(assessedUser.userId, criteria.getOrElse(config.user, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]], additionalProps)(metrics, config, httpUtil) - + logger.info("userDetails" + userDetails) //generateCertificateEvent if(userDetails.nonEmpty) { generateCertificateEvent(event, template, userDetails, enrolledUser, assessedUser, additionalProps, certName)(metrics, config, cache, httpUtil) @@ -210,10 +214,17 @@ trait IssueCertificateHelper { val lastName = Option(userDetails.getOrElse("lastName", "").asInstanceOf[String]).getOrElse("") def nullStringCheck(name:String):String = {if(StringUtils.equalsIgnoreCase("null", name)) "" else name} val recipientName = nullStringCheck(firstName).concat(" ").concat(nullStringCheck(lastName)).trim - val courseName = getCourseName(event.courseId)(metrics, config, cache, httpUtil) + val courseInfo: java.util.Map[String, AnyRef] = getCourseInfo(event.courseId)(metrics, config, cache, httpUtil) + val courseName = courseInfo.getOrDefault("courseName", "").asInstanceOf[String] val dateFormatter = new SimpleDateFormat("yyyy-MM-dd") val related = getRelatedData(event, enrolledUser, assessedUser, userDetails, additionalProps, certName, courseName)(config) - val providerName = getCourseOrganisation(event.courseId)(metrics, config, cache, httpUtil) + val parentCollections: List[String] = Option(courseInfo.get(config.parentCollections)) + .collect { + case list: java.util.List[_] => + list.asInstanceOf[java.util.List[String]].asScala.toList + } + .getOrElse(List.empty) + val eData = Map[String, AnyRef] ( "issuedDate" -> dateFormatter.format(enrolledUser.issuedOn), "data" -> List(Map[String, AnyRef]("recipientName" -> recipientName, "recipientId" -> event.userId)), @@ -229,10 +240,13 @@ trait IssueCertificateHelper { "basePath" -> config.certBasePath, "related" -> related, "name" -> certName, - "providerName" -> providerName, - "tag" -> event.batchId + "providerName" -> courseInfo.getOrDefault("providerName", "").asInstanceOf[String], + "tag" -> event.batchId, + "primaryCategory" -> courseInfo.getOrDefault("primaryCategory", "").asInstanceOf[String], + "parentCollections" -> parentCollections, + "coursePosterImage" -> courseInfo.getOrDefault("coursePosterImage", "").asInstanceOf[String], ) - + logger.info("Constructured eData from preProcessor : " + JSONUtil.serialize(eData)) ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObject(id= event.userId))) } @@ -252,4 +266,43 @@ trait IssueCertificateHelper { Map[String, Any]("batchId" -> event.batchId, "courseId" -> event.courseId, "type" -> certName) ++ locationProps ++ enrolledUser.additionalProps ++ assessedUser.additionalProps ++ userAdditionalProps ++ courseAdditionalProps } + + def getCourseInfo(courseId: String)(metrics: Metrics, config: CollectionCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = config.contentBasePath + config.contentReadApi + "/" + courseId + "?fields=name,parentCollections,primaryCategory,posterImage,organisation" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext.processEscapes(response.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(response.getOrElse(config.primaryCategory, "").asInstanceOf[String]).filter(_ >= ' ') + val posterImage: String = StringContext.processEscapes(response.getOrElse(config.posterImage, "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = response.getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + val orgData = response.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length - 1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap + } else { + val courseName = StringContext.processEscapes(courseMetadata.getOrElse(config.name, "").asInstanceOf[String]).filter(_ >= ' ') + val primaryCategory = StringContext.processEscapes(courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String]).filter(_ >= ' ') + val parentCollections = courseMetadata.getOrElse("parentcollections", new java.util.ArrayList()).asInstanceOf[java.util.ArrayList[String]] + val posterImage: String = StringContext.processEscapes(courseMetadata.getOrElse("posterimage", "").asInstanceOf[String]).filter(_ >= ' ') + val orgData = courseMetadata.get("organisation").toArray + val pm = orgData(0).toString + val providerName = pm.substring(1, pm.length - 1) + val courseInfoMap: java.util.Map[String, AnyRef] = new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("coursePosterImage", posterImage) + courseInfoMap.put("providerName", providerName) + courseInfoMap + } + } } diff --git a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala index f986cfc84..e69a9d8d8 100644 --- a/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala +++ b/credential-generator/collection-cert-pre-processor/src/main/scala/org/sunbird/job/collectioncert/task/CollectionCertPreProcessorConfig.scala @@ -79,6 +79,11 @@ class CollectionCertPreProcessorConfig(override val config: Config) extends Base val assessmentContentTypes = if(config.hasPath("assessment.metrics.supported.contenttype")) config.getStringList("assessment.metrics.supported.contenttype") else util.Arrays.asList("SelfAssess") val userAccBlockedErrCode = "UOS_USRRED0006" val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false - + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase+ "/content/v3/read/" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val posterImage: String = "posterImage" + val parentCollections: String = "parentCollections" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala index cb08ab00c..c7259302b 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Event.scala @@ -60,4 +60,9 @@ class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) def providerName: String = readOrDefault[String]("edata.providerName", "") + def primaryCategory: String = readOrDefault[String]("edata.primaryCategory", "") + + def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + + def coursePosterImage: String = readOrDefault[String]("edata.coursePosterImage", "") } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala index bb6baf196..5683709e2 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/domain/Models.scala @@ -64,3 +64,14 @@ case class UserEnrollmentData(batchId: String, case class Recipient(id: String, name: String, `type`: String) case class Training(id: String, name: String, `type`: String, batchId: String) case class Issuer(url: String, name: String, kid: String) +case class ActorObject(id: String = "Certificate Generator", `type`: String = "System") +case class EventObjectCourseCertificate(id: String, `type`: String = "GenerateCertificate") +case class EventContextCorseCertificate(pdata: Map[String, String] = Map("ver" -> "1.0", "id" -> "org.sunbird.learning.platform")) +case class BEJobRequestEvent(actor: ActorObject= ActorObject(), + eid: String = "BE_JOB_REQUEST", + edata: Map[String, AnyRef], + ets: Long = System.currentTimeMillis(), + context: EventContextCorseCertificate = EventContextCorseCertificate(), + mid: String = s"LMS.${UUID.randomUUID().toString}", + `object`: EventObjectCourseCertificate + ) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala index a8d3a92d6..7131bf5d3 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/CertificateGeneratorFunction.scala @@ -19,7 +19,7 @@ import org.sunbird.job.certgen.domain._ import org.sunbird.job.certgen.exceptions.ServerException import org.sunbird.job.certgen.task.CertificateGeneratorConfig import org.sunbird.job.exception.InvalidEventException -import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.util.{CassandraUtil, ElasticSearchUtil, HttpUtil, ScalaJsonUtil, JSONUtil} import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} import java.io.{File, IOException} @@ -27,8 +27,9 @@ import java.lang.reflect.Type import java.text.SimpleDateFormat import java.util import java.util.stream.Collectors -import java.util.{Base64, Date} +import java.util.{Base64, Date, UUID} import scala.collection.JavaConverters._ +import org.sunbird.job.certgen.domain.{ BEJobRequestEvent, EventObjectCourseCertificate} class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, storageService: StorageService, @transient var cassandraUtil: CassandraUtil = null) extends BaseProcessKeyedFunction[String, Event, String](config) { @@ -72,11 +73,11 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: if(certValidator.isNotIssued(event)(config, metrics, cassandraUtil)) { if(config.enableRcCertificate) generateCertificateUsingRC(event, context)(metrics) else generateCertificate(event, context)(metrics) - } else { metrics.incCounter(config.skippedEventCount) logger.info(s"Certificate already issued for: ${event.eData.getOrElse("userId", "")} ${event.related}") } + metrics.incCounter(config.successEventCount) } catch { case e: Exception => metrics.incCounter(config.failedEventCount) @@ -307,8 +308,9 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: val audit = ScalaJsonUtil.serialize(certificateAuditEvent) context.output(config.auditEventOutputTag, audit) logger.info("pushAuditEvent: certificate audit event success {}", audit) - context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, certMetaData.batchId, certMetaData.templateId, event.partition, event.offset)) - context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) + context.output(config.notifierOutputTag, NotificationMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, + certMetaData.batchId, certMetaData.templateId, event.partition, event.offset, event.providerName, event.coursePosterImage)) + //context.output(config.userFeedOutputTag, UserFeedMetaData(certMetaData.userId, certMetaData.courseName, issuedOn, certMetaData.courseId, event.partition, event.offset)) } else { metrics.incCounter(config.failedEventCount) throw new Exception(s"Update certificates to enrolments failed: ${event}") @@ -358,4 +360,23 @@ class CertificateGeneratorFunction(config: CertificateGeneratorConfig, httpUtil: } + def generateCourseCompletionEvent(event: Event) = { + val eData = Map[String, AnyRef]( + "userId" -> event.userId, + "batchId" -> event.batchId, + "courseId" -> event.courseId, + "parentCollections" -> event.parentCollections + ) + ScalaJsonUtil.serialize(BEJobRequestEvent(edata = eData, `object` = EventObjectCourseCertificate(id = event.userId))) + } + + /* + def createProgramCertPreProcessorEvent(event: Event, context: KeyedProcessFunction[String, Event, String]#Context) : Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Program Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${event.batchId}_${event.courseId}","type": "ProgramCertificateGeneration"},"edata": {"userIds": ["${event.userId}"],"action": "program_cert_pre_process","iteration": 1, "trigger": "auto-issue","batchId": "${event.batchId}","reIssue": false,"courseId": "${event.courseId}"}}""" + logger.info("Cert generator... Triggering program cert pre processor event : " + event) + context.output(config.generateProgramCertificateOutputTag, event) + } + */ } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala index 9711492cf..066ce1dcf 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/functions/NotifierFunction.scala @@ -20,7 +20,7 @@ import org.sunbird.job.{BaseProcessFunction, Metrics} import scala.collection.JavaConverters._ import scala.collection.mutable -case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long) +case class NotificationMetaData(userId: String, courseName: String, issuedOn: Date, courseId: String, batchId: String, templateId: String, partition: Int, offset: Long, courseProvider: String, coursePosterImage:String) class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[NotificationMetaData, String](config) { @@ -52,12 +52,13 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ val row = getNotificationTemplates(primaryFields, metrics) val certTemplate = row.getMap(config.cert_templates, com.google.common.reflect.TypeToken.of(classOf[String]), TypeTokens.mapOf(classOf[String], classOf[String])) - val url = config.learnerServiceBaseUrl + config.notificationEndPoint + val url = config.learnerServiceBaseUrl + config.newEmailTemplateNotificationEndPoint if (certTemplate != null && StringUtils.isNotBlank(metaData.templateId) && certTemplate.containsKey(metaData.templateId) && certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) { logger.info("notification template is present in the cert-templates object {}", certTemplate.get(metaData.templateId).containsKey(config.notifyTemplate)) + logger.info("Sending notification email. URL: {}", url) val notifyTemplate = getNotifyTemplateFromRes(certTemplate.get(metaData.templateId)) val ratingUrl = config.domainUrl + config.ratingMidPoint + metaData.courseId + config.ratingEndPoint + metaData.batchId val request = mutable.Map[String, AnyRef]("request" -> (notifyTemplate ++ mutable.Map[String, AnyRef]( @@ -66,8 +67,11 @@ class NotifierFunction(config: CertificateGeneratorConfig, httpUtil: HttpUtil, @ config.heldDate -> dateFormatter.format(metaData.issuedOn), config.recipientUserIds -> List[String](metaData.userId), config.ratingPageUrl -> ratingUrl, - config.body -> "email body"))) - + config.body -> "email body", + config.courseName -> metaData.courseName, + config.courseProvider -> metaData.courseProvider, + config.coursePosterImage -> metaData.coursePosterImage + ))) val response = httpUtil.post(url, ScalaJsonUtil.serialize(request)) if (response.status == 200) { metrics.incCounter(config.notifiedUserCount) diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala index e5793226d..08f4223d0 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorConfig.scala @@ -21,17 +21,18 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon // Kafka Topics Configuration val kafkaInputTopic: String = config.getString("kafka.input.topic") val kafkaAuditEventTopic: String = config.getString("kafka.output.audit.topic") + val kafkaProgramCertOutputTopic: String = config.getString("kafka.program.cert.output.topic") val enableSuppressException: Boolean = if(config.hasPath("enable.suppress.exception")) config.getBoolean("enable.suppress.exception") else false val enableRcCertificate: Boolean = if(config.hasPath("enable.rc.certificate")) config.getBoolean("enable.rc.certificate") else false - // Producers val certificateGeneratorAuditProducer = "collection-certificate-generator-audit-events-sink" override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") val notifierParallelism: Int = if(config.hasPath("task.notifier.parallelism")) config.getInt("task.notifier.parallelism") else 1 val userFeedParallelism: Int = if(config.hasPath("task.userfeed.parallelism")) config.getInt("task.userfeed.parallelism") else 1 + val generateProgramCertificateParallelism: Int = if(config.hasPath("task.programcert.parallelism")) config.getInt("task.programcert.parallelism") else 1 //ES configuration val esConnection: String = config.getString("es.basePath") @@ -164,5 +165,13 @@ class CertificateGeneratorConfig(override val config: Config) extends BaseJobCon val userFeedMsg: String = "You have earned a certificate! Download it from your profile page." val priorityValue = 1 val userFeedCount = "user-feed-count" - + val generateProgramCertificateOutputTagName: String = "generate-program-certificate-request" + val generateProgramCertificateOutputTag: OutputTag[String] = OutputTag[String](generateProgramCertificateOutputTagName) + + val generateProgramCertificateProducer = "generate-program-certificate-sink" + + val courseProvider: String ="courseProvider" + val coursePosterImage :String ="coursePosterImage" + + val newEmailTemplateNotificationEndPoint: String = "/v1/notification/email" } diff --git a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala index 5b7784e1f..00a409bcb 100755 --- a/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala +++ b/credential-generator/collection-certificate-generator/src/main/scala/org/sunbird/job/certgen/task/CertificateGeneratorStreamTask.scala @@ -55,6 +55,11 @@ class CertificateGeneratorStreamTask(config: CertificateGeneratorConfig, kafkaCo .uid("user-feed") .setParallelism(config.userFeedParallelism) + processStreamTask.getSideOutput(config.generateProgramCertificateOutputTag) + .addSink(kafkaConnector.kafkaStringSink(config.kafkaProgramCertOutputTopic)) + .name(config.generateProgramCertificateProducer) + .uid(config.generateProgramCertificateProducer) + .setParallelism(config.generateProgramCertificateParallelism) env.execute(config.jobName) } diff --git a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala index 2cd445226..7d63ef1f2 100755 --- a/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala +++ b/credential-generator/collection-certificate-generator/src/test/scala/org/sunbird/job/certgen/spec/NotifierFunctionTest.scala @@ -64,7 +64,7 @@ class NotifierFunctionTest extends BaseTestSpec { "NotifierFunction " should "should send notify user" in { implicit val notificationMetaTypeInfo: TypeInformation[NotificationMetaData] = TypeExtractor.getForClass(classOf[NotificationMetaData]) - new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0), null, metrics) + new NotifierFunction(notifierConfig, mockHttpUtil,cassandraUtil).processElement(NotificationMetaData("userId", "Course Name", new Date(), "do_11309999837886054415", "0131000245281587206", "template_01_dev_001",0, 0,"", ""), null, metrics) metrics.get(s"${notifierConfig.courseBatchdbReadCount}") should be(1) metrics.get(s"${notifierConfig.notifiedUserCount}") should be(1) metrics.get(s"${notifierConfig.skipNotifyUserCount}") should be(0) diff --git a/credential-generator/pom.xml b/credential-generator/pom.xml index 2c7fadf4a..a18fef683 100644 --- a/credential-generator/pom.xml +++ b/credential-generator/pom.xml @@ -14,6 +14,7 @@ collection-cert-pre-processor certificate-processor collection-certificate-generator + program-cert-pre-processor diff --git a/credential-generator/program-cert-pre-processor/pom.xml b/credential-generator/program-cert-pre-processor/pom.xml new file mode 100644 index 000000000..dfbaca15c --- /dev/null +++ b/credential-generator/program-cert-pre-processor/pom.xml @@ -0,0 +1,214 @@ + + + 4.0.0 + + org.sunbird + credential-generator + 1.0 + + program-cert-pre-processor + 1.0.0 + jar + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.version} + ${flink.version} + provided + + + org.sunbird + jobs-core + 1.0.0 + + + joda-time + joda-time + 2.10.6 + + + org.sunbird + jobs-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_${scala.version} + ${flink.version} + test + + + org.apache.flink + flink-runtime_${scala.version} + ${flink.version} + test + tests + + + org.apache.flink + flink-streaming-java_${scala.version} + ${flink.version} + test + tests + + + org.scalatest + scalatest_${scala.version} + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + it.ozimov + embedded-redis + 0.7.1 + test + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.job.programcert.task.ProgramCertPreProcessorTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + 11 + 11 + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + collection-complete-post-processor-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties b/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties new file mode 100644 index 000000000..cacd49c79 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/resources/log4j.properties @@ -0,0 +1,11 @@ +# log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.file=course-metrics-updater.log +log4j.appender.file.append=true +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.MaxFileSize=256KB +log4j.appender.file.MaxBackupIndex=4 +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf new file mode 100644 index 000000000..9f8116582 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/resources/program-cert-pre-processor.conf @@ -0,0 +1,38 @@ +include "base-config.conf" + +kafka { + input.topic = "dev.issue.program.certificate.request" + output.topic = "dev.issue.certificate.request" + output.failed.topic = "dev.issue.certificate.failed" + groupId = "dev-program-cert-pre-processor-group" +} + +task { + consumer.parallelism = 1 + parallelism = 1 + generate_certificate.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + user_enrolments.table = "user_enrolments" + course_batch.table = "course_batch" + assessment_aggregator.table = "assessment_aggregator" + user_activity_agg.table = "user_activity_agg" +} + +cert_domain_url="https://dev.sunbirded.org" +user_read_api = "/private/user/v1/read" +content_read_api = "/content/v3/read" + +service { + content.basePath = "http://localhost:9000" + learner.basePath = "http://localhost:9000" +} + +redis-meta { + host = localhost + port = 6379 +} +assessment.metrics.supported.contenttype = ["SelfAssess"] +enable.suppress.exception = true diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala new file mode 100644 index 000000000..9f3a46640 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Event.scala @@ -0,0 +1,29 @@ +package org.sunbird.job.programcert.domain + +import org.sunbird.job.domain.reader.JobRequest +import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig + +class Event(eventMap: java.util.Map[String, Any], partition: Int, offset: Long) extends JobRequest(eventMap, partition, offset) { + + def action:String = readOrDefault[String]("edata.action", "") + + def batchId: String = readOrDefault[String]("edata.batchId", "") + + def courseId: String = readOrDefault[String]("edata.courseId", "") + + def userId: String = readOrDefault[String]("edata.userId", "") + + def providerName: String = readOrDefault[String]("edata.providerName", "") + + def primaryCategory: String = readOrDefault[String]("edata.primaryCategory", "") + + def parentCollections: List[String] = readOrDefault[List[String]]("edata.parentCollections", List.empty[String]) + + + def eData: Map[String, AnyRef] = readOrDefault[Map[String, AnyRef]]("edata", Map[String, AnyRef]()) + + def isValid()(config: ProgramCertPreProcessorConfig): Boolean = { + config.programCertPreProcess.equalsIgnoreCase(action) && !batchId.isEmpty && !courseId.isEmpty && + !userId.isEmpty + } +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala new file mode 100644 index 000000000..635f881d9 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/domain/Models.scala @@ -0,0 +1,26 @@ +package org.sunbird.job.programcert.domain + +import java.util.{Date, UUID} + +case class EnrolledUser(userId: String, oldId: String = null, issuedOn: Date = null, additionalProps: Map[String, Any] = Map[String, Any]()) + +case class AssessedUser(userId: String, additionalProps: Map[String, Any] = Map[String, Any]()) + + +case class ActorObject(id: String = "Certificate Generator", `type`: String = "System") + +case class EventContext(pdata: Map[String, String] = Map("ver" -> "1.0", "id" -> "org.sunbird.learning.platform")) + + +case class EventObject(id: String, `type`: String = "GenerateCertificate") + +case class BEJobRequestEvent(actor: ActorObject= ActorObject(), + eid: String = "BE_JOB_REQUEST", + edata: Map[String, AnyRef], + ets: Long = System.currentTimeMillis(), + context: EventContext = EventContext(), + mid: String = s"LMS.${UUID.randomUUID().toString}", + `object`: EventObject + ) + +case class AssessmentUserAttempt(contentId: String, score: Double, totalScore: Double) \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala new file mode 100644 index 000000000..3220838bf --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/IssueCertificateHelper.scala @@ -0,0 +1,33 @@ +package org.sunbird.job.programcert.functions + +import java.text.SimpleDateFormat +import com.datastax.driver.core.querybuilder.QueryBuilder +import com.datastax.driver.core.{Row, TypeTokens} +import org.apache.commons.lang3.StringUtils +import org.slf4j.LoggerFactory +import org.sunbird.job.Metrics +import org.sunbird.job.cache.DataCache +import org.sunbird.job.programcert.domain.{AssessedUser, AssessmentUserAttempt, EnrolledUser, Event} +import org.sunbird.job.programcert.task.{ProgramCertPreProcessorConfig} +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} + +import scala.collection.JavaConverters._ + +trait IssueCertificateHelper { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) + def getAPICall(url: String, responseParam: String)(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil, metrics: Metrics): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil.deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()).asInstanceOf[Map[String, AnyRef]] + } else if (400 == response.status && response.body.contains(config.userAccBlockedErrCode)) { + metrics.incCounter(config.skippedEventCount) + logger.error(s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body) + Map[String, AnyRef]() + } else { + throw new Exception(s"Error from get API : ${url}, with response: ${response}") + } + } +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala new file mode 100644 index 000000000..753fe2516 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/functions/ProgramCertPreProcessorFn.scala @@ -0,0 +1,273 @@ +package org.sunbird.job.programcert.functions + +import com.datastax.driver.core.{Row, TypeTokens} +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.common.reflect.TypeToken +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper +import org.apache.flink.streaming.api.functions.KeyedProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.InvalidEventException +import org.sunbird.job.programcert.domain.Event +import org.sunbird.job.programcert.task.ProgramCertPreProcessorConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil} +import org.sunbird.job.{BaseProcessKeyedFunction, Metrics} + +import java.util.{Date, UUID} +import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} +import scala.collection.mutable +import scala.util.control.Breaks.{break, breakable} + +class ProgramCertPreProcessorFn(config: ProgramCertPreProcessorConfig, httpUtil: HttpUtil) + (implicit val stringTypeInfo: TypeInformation[String], + @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessKeyedFunction[String, Event, String](config) with IssueCertificateHelper { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorFn]) + private var cache: DataCache = _ + private var contentCache: DataCache = _ + lazy private val mapper: ObjectMapper = new ObjectMapper() + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + val redisConnect = new RedisConnect(config) + cache = new DataCache(config, redisConnect, config.collectionCacheStore, List()) + cache.init() + + val metaRedisConn = new RedisConnect(config, Option(config.metaRedisHost), Option(config.metaRedisPort)) + contentCache = new DataCache(config, metaRedisConn, config.contentCacheStore, List()) + contentCache.init() + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + override def metricsList(): List[String] = { + List(config.totalEventsCount, config.dbReadCount, config.dbUpdateCount, config.failedEventCount, config.skippedEventCount, config.successEventCount, + config.cacheHitCount, config.programCertIssueEventsCount) + } + + override def processElement(event: Event, + context: KeyedProcessFunction[String, Event, String]#Context, + metrics: Metrics): Unit = { + try { + val getParentIdForCourse = event.parentCollections + if (!getParentIdForCourse.isEmpty) { + val enrolmentRecords = getAllEnrolments(event.userId)(metrics) + for (courseParentId <- getParentIdForCourse) { + val programEnrollmentRow = getEnrollmentRecord(enrolmentRecords, courseParentId) + //if enrolled into program + if (programEnrollmentRow.isDefined && programEnrollmentRow.get.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty) { + val programHierarchy = getProgramChildren(courseParentId)(metrics, config, contentCache, httpUtil) + if (!programHierarchy.isEmpty) { + val batchId: String = programEnrollmentRow.get.getString(config.dbBatchId) + val contentDataForProgram = programHierarchy.get(config.childrens).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] + val leafNodeMap = mutable.Map[String, Int]() + + var isProgramCertificateToBeGenerated: Boolean = true; + var programCompletedOn: Date = null + for (childNode <- contentDataForProgram) { + val primaryCategory = childNode.get(config.primaryCategory).asInstanceOf[String] + if (config.allowedPrimaryCategoryForProgram.contains(primaryCategory)) { + val courseId: String = childNode.get(config.identifier).asInstanceOf[String] + val userId: String = event.userId + val courseEnrollmentRow = getEnrollmentRecord(enrolmentRecords, courseId) + val isCertificateIssued = courseEnrollmentRow.isDefined && !courseEnrollmentRow.get.getList(config.issuedCertificates, TypeTokens.mapOf(classOf[String], classOf[String])).isEmpty + logger.info("Is Certificate Available for courseId: " + courseId + " userId:" + userId + " :" + isCertificateIssued) + var courseCompletedOn: Date = null; + if (isCertificateIssued) { + courseCompletedOn = courseEnrollmentRow.get.getTimestamp("completedon") + if (programCompletedOn == null) { + programCompletedOn = courseCompletedOn + } else if (programCompletedOn.before(courseCompletedOn)) { + programCompletedOn = courseCompletedOn + } + } + + breakable { + if (!isCertificateIssued) { + isProgramCertificateToBeGenerated = false; + break + } else { + val leafNodes = childNode.get(config.leafNodes).asInstanceOf[java.util.List[String]] + for (leafNode <- leafNodes) { + leafNodeMap += (leafNode -> 2) + } + } + } + } + } + if (!leafNodeMap.isEmpty) { + val programContentStatus = Option(programEnrollmentRow.get.getMap( + config.contentStatus, TypeToken.of(classOf[String]), TypeToken.of(classOf[Integer]))).head + var progressCount: Integer = Option(programEnrollmentRow.get.getInt(config.progress)).head + + var updateCount = 0 + + for ((key, value) <- leafNodeMap) { + // Check if the key is present in leafNodeMap + if (programContentStatus.get(key) != null) { + if (programContentStatus.get(key) != 2) { + // Update progress in contentStatus for the matching key + programContentStatus.put(key, value) + updateCount += 1 + } + } else { + programContentStatus.put(key, value) + updateCount += 1 + } + } + + // Update the progress with the total update count + progressCount += updateCount + var status: Int = 1 + val leafNodesForProgram = programHierarchy.get(config.leafNodes).asInstanceOf[java.util.List[String]] + if (progressCount == leafNodesForProgram.size()) { + status = 2 + } else { + isProgramCertificateToBeGenerated = false + } + updateEnrolment(event.userId, batchId, courseParentId, programContentStatus, status, progressCount, programCompletedOn)(metrics) + } + + if (isProgramCertificateToBeGenerated) { + //Add kafka event to generate Certificate for Program + logger.info("Adding the kafka event for programId: " + courseParentId) + createIssueCertEventForProgram(courseParentId, event.userId, batchId, context)(metrics) + } + } + } + } + } + } catch { + case ex: Exception => { + throw new InvalidEventException(ex.getMessage, Map("partition" -> event.partition, "offset" -> event.offset), ex) + } + } + logger.info("Inside the Process ElementForProgram"); + } + + def getProgramChildren(programId: String)(metrics: Metrics, config: ProgramCertPreProcessorConfig, cache: DataCache, httpUtil: HttpUtil): java.util.Map[String, AnyRef] = { + val query = QueryBuilder.select(config.Hierarchy).from(config.contentHierarchyKeySpace, config.contentHierarchyTable) + .where(QueryBuilder.eq(config.identifier, programId)) + val row = cassandraUtil.find(query.toString) + if (CollectionUtils.isNotEmpty(row)) { + val hierarchy = row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] + if (StringUtils.isNotBlank(hierarchy)) + mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) + else new java.util.HashMap[String, AnyRef]() + } + else new java.util.HashMap[String, AnyRef]() + } + + private def getCourseEnrollment(columns: Map[String, AnyRef])(implicit metrics: Metrics): Row = { + logger.info("primary columns {}", columns) + val selectWhere = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + columns.map(col => { + col._2 match { + case value: List[Any] => + selectWhere.and(QueryBuilder.in(col._1, value.asJava)) + case _ => + selectWhere.and(QueryBuilder.eq(col._1, col._2)) + } + }) + logger.info("select query {}", selectWhere.toString) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + if (row.size() == 1) { + row.asScala.get(0) + } else { + logger.error("More than one certificate" + columns) + null + } + } else { + logger.error("No Certificate Available" + columns) + null + } + } + + def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.keyspace, config.userEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) + .and(QueryBuilder.eq(config.dbCourseId, programId)) + metrics.incCounter(config.dbReadCount) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + if (row.size() == 1) { + row.asScala.get(0) + } else { + logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) + null + } + } else { + logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) + null + } + } + + def updateEnrolment(userId: String, batchId: String, programId: String, contentStatus: java.util.Map[String, Integer], status: Int, progress: Int, programCompletedOn: Date)(implicit metrics: Metrics): Unit = { + logger.info("Enrolment updated for userId: " + userId + " batchId: " + batchId) + val updateQuery = QueryBuilder.update(config.keyspace, config.userEnrolmentsTable) + .`with`(QueryBuilder.set("status", status)) + .and(QueryBuilder.set("progress", progress)) + .and(QueryBuilder.set("contentstatus", contentStatus)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + if (status == 2) { + updateQuery.and(QueryBuilder.set("completedon", programCompletedOn)) + } + updateQuery.where(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", programId)) + .and(QueryBuilder.eq("batchid", batchId)) + + val result = cassandraUtil.upsert(updateQuery.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed" + updateQuery.toString + logger.error(msg) + throw new Exception(msg) + } + } + + def createIssueCertEventForProgram(programId: String, userId: String, batchId: String, context: KeyedProcessFunction[String, Event, String]#Context)(implicit metrics: Metrics): Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Program Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${batchId}_${programId}","type": "ProgramCertificateGeneration"},"edata": {"userIds": ["${userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${batchId}","reIssue": false,"courseId": "${programId}"}}""" + logger.info("o/p event: " + event) + context.output(config.generateCertificateOutputTag, event) + metrics.incCounter(config.programCertIssueEventsCount) + } + + def getAllEnrolments(userId: String)(implicit metrics: Metrics): java.util.List[Row] = { + val selectWhere: Select.Where = QueryBuilder.select(config.dbUserId, config.dbCourseId, config.dbBatchId, config.contentStatus, config.progress, config.issuedCertificates, "completedon", "active") + .from(config.keyspace, config.userEnrolmentsTable).where() + selectWhere.and(QueryBuilder.eq(config.dbUserId, userId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.find(selectWhere.toString) + } + + def getEnrollmentRecord(enrollList: java.util.List[Row], courseId: String): Option[Row] = { + if(null != enrollList) { + enrollList.asScala.find { row => + val courseid = row.getString("courseid") + val active = row.getBool("active") + courseid == courseId && active + } + } else { + None + } + } +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala new file mode 100644 index 000000000..f817f773d --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorConfig.scala @@ -0,0 +1,76 @@ +package org.sunbird.job.programcert.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.job.BaseJobConfig + +class ProgramCertPreProcessorConfig(override val config: Config) extends BaseJobConfig(config, "program-cert-pre-processor") { + + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + + //Redis config + val collectionCacheStore: Int = 0 + val contentCacheStore: Int = 5 + val metaRedisHost: String = config.getString("redis-meta.host") + val metaRedisPort: Int = config.getInt("redis-meta.port") + + //kafka config + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaOutputTopic: String = config.getString("kafka.output.topic") + val certificatePreProcessorConsumer: String = "program-cert-pre-processor-consumer" + val generateCertificateProducer = "generate-certificate-sink" + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val generateCertificateParallelism: Int = config.getInt("task.generate_certificate.parallelism") + + //Tags + val generateCertificateOutputTagName = "generate-certificate-request" + val generateCertificateOutputTag: OutputTag[String] = OutputTag[String](generateCertificateOutputTagName) + + //Cassandra config + val dbHost: String = config.getString("lms-cassandra.host") + val dbPort: Int = config.getInt("lms-cassandra.port") + val keyspace: String = config.getString("lms-cassandra.keyspace") + val userEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") + val dbBatchId = "batchId" + val dbCourseId = "courseid" + val dbUserId = "userid" + val contentHierarchyTable: String = "content_hierarchy" + val contentHierarchyKeySpace: String = "dev_hierarchy_store" + val Hierarchy: String = "hierarchy" + val childrens: String = "children" + val batches: String = "batches" + + //API URL + val contentBasePath = config.getString("service.content.basePath") + val learnerBasePath = config.getString("service.learner.basePath") + val userReadApi = config.getString("user_read_api") + val contentReadApi = "/content/v4/read" + + // Metric List + val totalEventsCount = "total-events-count" + val successEventCount = "success-events-count" + val failedEventCount = "failed-events-count" + val skippedEventCount = "skipped-event-count" + val dbReadCount = "db-read-count" + val dbUpdateCount = "db-update-count" + val cacheHitCount = "cache-hit-cout" + val programCertIssueEventsCount = "program-cert-issue-events-count" + + //Constants + val status: String = "status" + val name: String = "name" + val defaultHeaders = Map[String, String]("Content-Type" -> "application/json") + val identifier: String = "identifier" + val userAccBlockedErrCode = "UOS_USRRED0006" + val programCertPreProcess: String = "program_cert_pre_process" + val parentCollections: String = "parentCollections" + val issuedCertificates: String = "issued_certificates" + val primaryCategory: String = "primaryCategory" + val leafNodes: String = "leafNodes" + val contentStatus: String = "contentstatus" + val progress: String = "progress" + val allowedPrimaryCategoryForProgram = List[String]("Course") + +} diff --git a/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala new file mode 100644 index 000000000..41a7bf873 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/main/scala/org/sunbird/job/programcert/task/ProgramCertPreProcessorTask.scala @@ -0,0 +1,58 @@ +package org.sunbird.job.programcert.task + +import java.io.File +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment +import org.slf4j.LoggerFactory +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.programcert.domain.Event +import org.sunbird.job.programcert.functions.ProgramCertPreProcessorFn +import org.sunbird.job.util.{FlinkUtil, HttpUtil} + +class ProgramCertPreProcessorTask(config: ProgramCertPreProcessorConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramCertPreProcessorTask]) + + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + val source = kafkaConnector.kafkaJobRequestSource[Event](config.kafkaInputTopic) + logger.info("This is under process for task") + val progressStream = + env.addSource(source).name(config.certificatePreProcessorConsumer) + .uid(config.certificatePreProcessorConsumer).setParallelism(config.kafkaConsumerParallelism) + .rebalance + .keyBy(new ProgramCertPreProcessorKeySelector()) + .process(new ProgramCertPreProcessorFn(config, httpUtil)) + .name("program-cert-pre-processor").uid("program-cert-pre-processor") + .setParallelism(config.parallelism) + + progressStream.getSideOutput(config.generateCertificateOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaOutputTopic)) + .name(config.generateCertificateProducer).uid(config.generateCertificateProducer).setParallelism(config.generateCertificateParallelism) + env.execute(config.jobName) + } +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster + +object ProgramCertPreProcessorTask { + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("program-cert-pre-processor.conf").withFallback(ConfigFactory.systemEnvironment())) + val certificatePreProcessorConfig = new ProgramCertPreProcessorConfig(config) + val kafkaUtil = new FlinkKafkaConnector(certificatePreProcessorConfig) + val httpUtil = new HttpUtil() + val task = new ProgramCertPreProcessorTask(certificatePreProcessorConfig, kafkaUtil, httpUtil) + task.process() + } +} + +class ProgramCertPreProcessorKeySelector extends KeySelector[Event, String] { + override def getKey(event: Event): String = Set(event.userId, event.courseId, event.batchId).mkString("_") +} \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml b/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml new file mode 100644 index 000000000..e81294323 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/logback-test.xml @@ -0,0 +1,16 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + \ No newline at end of file diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/test.conf b/credential-generator/program-cert-pre-processor/src/test/resources/test.conf new file mode 100644 index 000000000..efad514b6 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/test.conf @@ -0,0 +1,45 @@ +include "base-test.conf" + +kafka { + input.topic = "flink.issue.certificate.request" + output.topic = "flink.generate.certificate.request" + output.failed.topic = "flink.issue.certificate.failed" + groupId = "flink-collection-cert-pre-processor-group" +} + +task { + consumer.parallelism = 1 + parallelism = 1 + generate_certificate.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + user_enrolments.table = "user_enrolments" + course_batch.table = "course_batch" + assessment_aggregator.table = "assessment_aggregator" + user_activity_agg.table = "user_activity_agg" + host = "localhost" + port = "9142" +} + +dp-redis { + host = localhost + port = 6340 + database.index = 5 +} + +cert_domain_url="https://dev.sunbirded.org" +user_read_api = "/private/user/v1/read" +content_read_api = "/content/v3/read" + +service { + content.basePath = "http://localhost:9000/content" + learner.basePath = "http://localhost:9000/learner" +} + +redis-meta { + host = localhost + port = 6379 +} +assessment.metrics.supported.contenttype = ["selfAssess"] diff --git a/credential-generator/program-cert-pre-processor/src/test/resources/test.cql b/credential-generator/program-cert-pre-processor/src/test/resources/test.cql new file mode 100644 index 000000000..5a5b056cf --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/resources/test.cql @@ -0,0 +1,85 @@ +CREATE KEYSPACE IF NOT EXISTS sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE IF NOT EXISTS sunbird_courses.course_batch ( + courseid text, + batchid text, + cert_templates map>>, + createdby text, + createddate text, + createdfor list, + description text, + enddate text, + enrollmentenddate text, + enrollmenttype text, + mentors list, + name text, + startdate text, + status int, + updateddate text, + PRIMARY KEY (courseid, batchid) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_enrolments ( + userid text, + courseid text, + batchid text, + active boolean, + addedby text, + certificates list>>, + completedon timestamp, + completionpercentage int, + contentstatus map, + datetime timestamp, + issued_certificates list>>, + enrolleddate text, + lastreadcontentid text, + lastreadcontentstatus int, + progress int, + status int, + PRIMARY KEY (userid, courseid, batchid) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator ( + user_id text, + course_id text, + batch_id text, + content_id text, + attempt_id text, + created_on timestamp, + grand_total text, + last_attempted_on timestamp, + total_max_score double, + total_score double, + updated_on timestamp, + PRIMARY KEY ((user_id, course_id), batch_id, content_id, attempt_id) +); + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg ( + activity_id text, + user_id text, + activity_type text, + context_id text, + agg Map, + aggregates Map, + agg_last_updated Map, + PRIMARY KEY ((activity_type, activity_id), context_id, user_id) +); + + +//event 1 +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('do_11309999837886054415','0131000245281587206',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}, "assessment": {"score": {">=": 80}}}', 'identifier': 'template_01_dev_001', 'url': 'template-url.svg', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course Completion Certificate', 'notifyTemplate': '{"subject":"Completion certificate","stateImgUrl":"https://sunbirddev.blob.core.windows.net/orgemailtemplate/img/File-0128212938260643843.png","regardsperson":"Chairperson","regards":"Minister of Gujarat","emailTemplateType":"defaultCertTemp"}', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, issued_certificates, completedon, status, active) VALUES ('user001','do_11309999837886054415','0131000245281587206',[{'identifier': 'certificateId', 'lastIssuedOn': '2019-08-21', 'name': 'Course Completion Certificate', 'token': 'P4L3Y9'}], toTimeStamp(toDate(now())), 2, true); +INSERT INTO sunbird_courses.assessment_aggregator(user_id, course_id, batch_id, content_id, attempt_id, total_max_score, total_score) VALUES ('user001','do_11309999837886054415','0131000245281587206', 'content_001', 'attempt_001', 1, 1); +INSERT INTO sunbird_courses.user_activity_agg(activity_id, user_id, activity_type, context_id, aggregates) VALUES ('do_11309999837886054415', 'user001', 'Course', 'cb:0131000245281587206', {'score:content_001': 1, 'max_score:content_001': 1}); +//event 2 +INSERT INTO sunbird_courses.course_batch(courseid, batchid,cert_templates) VALUES ('do_11309999837886054416','0131000245281587207',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}}','url': 'template-url.svg', 'identifier': 'template_01_dev_001', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course merit certificate', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid,completedon) VALUES ('user002','do_11309999837886054416','0131000245281587207', toTimeStamp(toDate(now()))); +//empty cert_template +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('course_002','batch_002',{}); + +//event 3 +INSERT INTO sunbird_courses.course_batch(courseid, batchid, cert_templates) VALUES ('course_003','batch_003',{'template_01_dev_001':{'criteria': '{"enrollment":{"status":2}, "assessment": {"score": {">=": 80}}}', 'identifier': 'template_01_dev_001', 'url': 'template-url.svg', 'issuer': '{"name":"Gujarat Council of Educational Research and Training","publicKey":["7","8"],"url":"https://gcert.gujarat.gov.in/gcert/"}', 'name': 'Course merit certificate', 'notifyTemplate': '{"subject":"Completion certificate","stateImgUrl":"https://sunbirddev.blob.core.windows.net/orgemailtemplate/img/File-0128212938260643843.png","regardsperson":"Chairperson","regards":"Minister of Gujarat","emailTemplateType":"defaultCertTemp"}', 'signatoryList': '[{"image":"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg","name":"CEO Gujarat","id":"CEO","designation":"CEO"}]'}}); +//user with empty issued certificate +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, issued_certificates, completedon, status, active) VALUES ('user003','course_003','batch_003',[], toTimeStamp(toDate(now())), 2, true); +INSERT INTO sunbird_courses.assessment_aggregator(user_id, course_id, batch_id, content_id, attempt_id, total_max_score, total_score) VALUES ('user003','course_003','batch_003', 'content_001', 'attempt_001', 1, 1); +INSERT INTO sunbird_courses.user_activity_agg(activity_id, user_id, activity_type, context_id, agg) VALUES ('course_003', 'user003', 'Course', 'cb:batch_003', {'score:content_001': 1, 'max_score:content_001': 1}); diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala new file mode 100644 index 000000000..945ddd9e6 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/fixture/EventFixture.scala @@ -0,0 +1,15 @@ +package org.sunbird.job.programcert.fixture + +object EventFixture { + + val EVENT_1: String = """{"eid": "BE_JOB_REQUEST","ets": 1621833310477,"mid": "LP.1621833310477.429e795f-6d3e-4a11-8754-685984d62d10","actor": {"id": "Course Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "0131000245281587206_do_11309999837886054416","type": "CourseCertificateGeneration"},"edata": {"userIds": ["user001"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "0131000245281587206","reIssue": true,"courseId": "do_11309999837886054415"}}""" + val USER_1: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"user","lastName":"name","rootOrgId": "Org001","userLocations":[{"code":"29","name":"Karnataka","id":"027f81d8-0a2c-4fc6-96ac-59fe4cea3abf","type":"state","parentId":null},{"code":"2920","name":"BENGALURU URBAN SOUTH","id":"fa17379e-f8d5-4403-ae64-9e339f1dd599","type":"district","parentId":"027f81d8-0a2c-4fc6-96ac-59fe4cea3abf"}],"rootOrg":{"keys":{}}}}}""" + val CONTENT_1: String = """{"id":"api.content.read","ver":"3.0","ts":"2021-05-27T10:31:33ZZ","params":{"resmsgid":"316b05dd-df1a-4867-968a-042bb06a710f","msgid":null,"err":null,"status":"successful","errmsg":null},"responseCode":"OK","result":{"content":{"objectType":"Content","primaryCategory":"Course","contentType":"Course","identifier":"do_11309999837886054416","languageCode":["en"],"name":"test Course"}}}""" + val TEMPLATE_1: String = """{"criteria":"{\"enrollment\":{\"status\":2},\"assessment\":{\"score\":{\">=\":50}}}","identifier":"template_svg_04-prad","issuer":"{\"name\":\"Gujarat Council of Educational Research and Training\",\"url\":\"https://gcert.gujarat.gov.in/gcert/\"}","name":"Course Completion Certificate","signatoryList":"[{\"image\":\"https://cdn.pixabay.com/photo/2014/11/09/08/06/signature-523237__340.jpg\",\"name\":\"CEO Gujarat\",\"id\":\"CEO\",\"designation\":\"CEO\"}]","url":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/template_svg_04-prad/artifact/template-1.svg","additionalProps": "{\"enrollment\":[\"completedOn\"],\"location\":[\"state\",\"district\",\"school\"],\"assessment\":[\"score\"],\"course\":[\"name\"]}"}""" + + val USER_2_EMPTY_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Rajesh","lastName":"","rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + val USER_3_NULL_VALUE_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Suresh","lastName":null,"rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + val USER_4_NULL_STRING_VALUE_LASTNAME: String = """{"id":".private.user.v1.read.c4cc494f-04c3-49f3-b3d5-7b1a1984abad","ver":"private","ts":"2021-05-27 10:25:05:836+0000","params":{"resmsgid":null,"msgid":"8e27cbf5-e299-43b0-bca7-8347f7e5abcf","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"firstName":"Manju","lastName":"null","rootOrgId": "Org001","rootOrg":{"keys":{}}}}}""" + + +} diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala new file mode 100644 index 000000000..4054696fe --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessFnTestSpec.scala @@ -0,0 +1,6 @@ +package org.sunbird.job.programcert.function.spec + +import org.sunbird.spec.BaseTestSpec + +class ProgramCertPreProcessFnTestSpec extends BaseTestSpec { +} diff --git a/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala new file mode 100644 index 000000000..03077a9d8 --- /dev/null +++ b/credential-generator/program-cert-pre-processor/src/test/scala/org/sunbird/job/programcert/function/spec/ProgramCertPreProcessorTaskSpec.scala @@ -0,0 +1,9 @@ +package org.sunbird.job.programcert.function.spec + +import org.scalatest.DoNotDiscover +import org.sunbird.spec.BaseTestSpec + +@DoNotDiscover +class ProgramCertPreProcessorTaskSpec extends BaseTestSpec { + +} \ No newline at end of file diff --git a/jobs-distribution/pom.xml b/jobs-distribution/pom.xml index bbc03d9b5..ca66efe24 100644 --- a/jobs-distribution/pom.xml +++ b/jobs-distribution/pom.xml @@ -122,6 +122,18 @@ 1.0.0 jar + + org.sunbird + program-cert-pre-processor + 1.0.0 + jar + + + org.sunbird + program-activity-aggregate-updater + 1.0.0 + jar + diff --git a/pom.xml b/pom.xml index 84e84ec96..18fe6e0a3 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,7 @@ audit-event-generator metrics-data-transformer qrcode-image-generator + program-activity-aggregate-updater diff --git a/post-publish-processor/src/main/resources/post-publish-processor.conf b/post-publish-processor/src/main/resources/post-publish-processor.conf index d214660e0..2d7391099 100644 --- a/post-publish-processor/src/main/resources/post-publish-processor.conf +++ b/post-publish-processor/src/main/resources/post-publish-processor.conf @@ -17,11 +17,13 @@ task { shallow_copy.parallelism = 1 link_dialcode.parallelism = 1 batch_create.parallelism = 1 + post-publish-relation-update.parallelism = 1 } lms-cassandra { keyspace = "sunbird_courses" batchTable = "course_batch" + hierarchyStoreKeySpace = "dev_hierarchy_store" } dialcode-cassandra { @@ -34,6 +36,7 @@ service { lms.basePath = "http://11.2.6.6/lms" learning_service.basePath = "http://11.2.4.22:8080/learning-service" dial.basePath = "https://dev.sunbirded.org/dial/" + content.basePath = "http://11.2.6.6/content" } dialcode { diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala index f37a06c63..4bc1a7614 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishEventRouter.scala @@ -4,8 +4,9 @@ import com.google.gson.reflect.TypeToken import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.functions.ProcessFunction import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} import org.sunbird.job.postpublish.domain.Event -import org.sunbird.job.postpublish.helpers.{BatchCreation, DialHelper, ShallowCopyPublishing} +import org.sunbird.job.postpublish.helpers.{BatchCreation, DialHelper, PostPublishRelationUpdater, ShallowCopyPublishing} import org.sunbird.job.postpublish.task.PostPublishProcessorConfig import org.sunbird.job.util.{CassandraUtil, HttpUtil, Neo4JUtil} import org.sunbird.job.{BaseProcessFunction, Metrics} @@ -17,7 +18,7 @@ case class PublishMetadata(identifier: String, contentType: String, mimeType: St class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpUtil, @transient var neo4JUtil: Neo4JUtil = null, @transient var cassandraUtil: CassandraUtil = null) - extends BaseProcessFunction[Event, String](config) with ShallowCopyPublishing with BatchCreation with DialHelper { + extends BaseProcessFunction[Event, String](config) with ShallowCopyPublishing with BatchCreation with DialHelper with PostPublishRelationUpdater { private[this] val logger = LoggerFactory.getLogger(classOf[PostPublishEventRouter]) val mapType: Type = new TypeToken[java.util.Map[String, AnyRef]]() {}.getType @@ -52,6 +53,10 @@ class PostPublishEventRouter(config: PostPublishProcessorConfig, httpUtil: HttpU val dialCodeDetails = getDialCodeDetails(identifier, event)(neo4JUtil, config) if (!dialCodeDetails.isEmpty) context.output(config.linkDIALCodeOutTag, dialCodeDetails) + + //Process Post Publish Relation Update + context.output(config.postPublishRelationUpdateOutTag, identifier) + } else { metrics.incCounter(config.skippedEventCount) logger.info(s"Event not qualified for publishing for Identifier : ${event.collectionId}.") diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala new file mode 100644 index 000000000..a2c96daee --- /dev/null +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/functions/PostPublishRelationUpdaterFunction.scala @@ -0,0 +1,194 @@ +package org.sunbird.job.postpublish.functions + +import com.datastax.driver.core.querybuilder.QueryBuilder +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.configuration.Configuration +import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.apache.http.client.methods.HttpPatch +import org.apache.http.entity.{ContentType, StringEntity} +import org.apache.http.impl.client.HttpClients +import org.apache.http.{HttpResponse, StatusLine} +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.APIException +import org.sunbird.job.postpublish.helpers.PostPublishRelationUpdater +import org.sunbird.job.postpublish.task.PostPublishProcessorConfig +import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil, JSONUtil} +import org.sunbird.job.{BaseProcessFunction, Metrics} + +import java.time.format.DateTimeFormatter +import java.time.{ZoneId, ZonedDateTime} +import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.{`collection AsScalaIterable`, `seq AsJavaList`} +import scala.collection.mutable.ListBuffer + +/** @author + * mahesh.vakkund + */ +class PostPublishRelationUpdaterFunction( + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + @transient var cassandraUtil: CassandraUtil = null +) extends BaseProcessFunction[String, String](config) + with PostPublishRelationUpdater { + + private[this] val logger = + LoggerFactory.getLogger(classOf[PostPublishRelationUpdaterFunction]) + lazy private val mapper: ObjectMapper = new ObjectMapper() + private var cache: DataCache = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + val redisConnect = new RedisConnect(config) + cache = + new DataCache(config, redisConnect, config.contentCacheStore, List()) + cache.init() + } + + override def close(): Unit = { + cassandraUtil.close() + cache.close() + super.close() + } + + private def postPublishRelationUpdate( + identifier: String + )(implicit + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + cassandraUtil: CassandraUtil, + metrics: Metrics + ): Unit = { + val programHierarchy = getProgramHierarchy( + identifier + )(metrics, config, cache, httpUtil) + if (programHierarchy.isEmpty) { + logger.info( + "PostPublishRelationUpdaterFunction :: Failed to get program Hierarchy." + ) + return + } + + val childrenList = programHierarchy.get(config.children).asInstanceOf[java.util.List[java.util.HashMap[String, AnyRef]]] + for (childNode <- childrenList) { + val primaryCategory: String = childNode.get(config.primaryCategory).asInstanceOf[String] + val childId: String = childNode.get("identifier").asInstanceOf[String] + if (primaryCategory.equalsIgnoreCase("Course")) { + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(childId)(metrics, config, cache, httpUtil) + var versionKey: String = contentObj.getOrDefault(config.versionKey, "").asInstanceOf[String] + logger.info("Child Course Id: " + childId + ", Info: " + JSONUtil.serialize(contentObj)) + + // Use Option to safely handle null values + val parentCollections: List[String] = Option(contentObj.get(config.parentCollections)) + .collect { + case list: java.util.List[_] => + list.asInstanceOf[java.util.List[String]].asScala.toList + } + .getOrElse(List.empty) + + // Update parentCollections if identifier is not present + val updatedParentCollections = if (!parentCollections.contains(identifier)) { + parentCollections :+ identifier + } else { + parentCollections + } + + val requestData: Map[String, Any] = Map( + "request" -> Map( + "content" -> Map( + "versionKey" -> versionKey, + "parentCollections" -> updatedParentCollections + ) + )) + val jsonString: String = JSONUtil.serialize(requestData) + logger.info("Calling content update with body: " + jsonString) + val patchRequest = new HttpPatch( + config.contentSystemUpdatePath + childId + ) + patchRequest.setEntity( + new StringEntity(jsonString, ContentType.APPLICATION_JSON) + ) + val httpClient = HttpClients.createDefault() + val response: HttpResponse = httpClient.execute(patchRequest) + val statusLine: StatusLine = response.getStatusLine + val statusCode: Int = statusLine.getStatusCode + if (statusCode == 200) { + logger.info("Processed the request.") + } else { + logger.error( + "Received error response for system update API. Response: " + JSONUtil + .serialize(response) + ) + } + } + } + } + + def getProgramHierarchy(programId: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val query = QueryBuilder + .select(config.Hierarchy) + .from(config.hierarchyStoreKeySpace, config.contentHierarchyTable) + .where(QueryBuilder.eq(config.identifier, programId)) + val row = cassandraUtil.find(query.toString) + if (CollectionUtils.isNotEmpty(row)) { + val hierarchy = + row.asScala.head.getObject(config.Hierarchy).asInstanceOf[String] + if (StringUtils.isNotBlank(hierarchy)) + mapper.readValue(hierarchy, classOf[java.util.Map[String, AnyRef]]) + else new java.util.HashMap[String, AnyRef]() + } else new java.util.HashMap[String, AnyRef]() + } + + override def processElement( + identifier: String, + context: ProcessFunction[String, String]#Context, + metrics: Metrics + ): Unit = { + val isValidProgram: Boolean = + verifyPrimaryCategory(identifier)(metrics, config, httpUtil, cache) + if (isValidProgram) { + metrics.incCounter(config.postPublishRelationUpdateEventCount) + logger.info( + "PostPublishRelationUpdaterFunction:: started for Content : " + identifier + ) + try { + postPublishRelationUpdate(identifier)( + config, + httpUtil, + cassandraUtil, + metrics + ) + metrics.incCounter(config.postPublishRelationUpdateSuccessCount) + logger.info( + "PostPublishRelationUpdaterFunction:: Completed for ContentId : " + identifier + ) + } catch { + case ex: Throwable => + logger.error( + s"Error while processing message for identifier : ${identifier}.", + ex + ) + metrics.incCounter(config.postPublishRelationUpdateFailureCount) + throw ex + } + } else { + logger.info( + "PostPublishRelationUpdaterFunction:: Nothing to do for ContentId : " + identifier + ) + } + } + + override def metricsList(): List[String] = { + List(config.postPublishRelationUpdateEventCount, + config.postPublishRelationUpdateSuccessCount, + config.postPublishRelationUpdateFailureCount) + } +} \ No newline at end of file diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala index 657e56556..3a66ccf67 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/BatchCreation.scala @@ -9,7 +9,6 @@ import org.sunbird.job.util.{CassandraUtil, HttpUtil, JSONUtil, Neo4JUtil} import java.util import scala.collection.JavaConverters._ -import scala.collection.JavaConverters trait BatchCreation { diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala new file mode 100644 index 000000000..8165e3c3e --- /dev/null +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/helpers/PostPublishRelationUpdater.scala @@ -0,0 +1,134 @@ +package org.sunbird.job.postpublish.helpers + +import org.apache.flink.configuration.Configuration +import org.slf4j.LoggerFactory +import org.sunbird.job.Metrics +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.exception.APIException +import org.sunbird.job.postpublish.domain.Event +import org.sunbird.job.postpublish.task.PostPublishProcessorConfig +import org.sunbird.job.util._ + +/** @author + * mahesh.vakkund + */ +trait PostPublishRelationUpdater { + + private[this] val logger = + LoggerFactory.getLogger(classOf[PostPublishRelationUpdater]) + + def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (!contentObj.isEmpty) { + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory == "Program" + || primaryCategory == "Curated Program" + || primaryCategory == "Blended Program")) { + isValidProgram = true + } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) + } + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: PostPublishProcessorConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + "/" + courseId + "?fields=identifier,name,versionKey,parentCollections,primaryCategory" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + logger.info("Content read response" + JSONUtil.serialize(response)) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val versionKey = StringContext + .processEscapes( + response.getOrElse(config.versionKey, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse("parentCollections", List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put(config.versionKey, versionKey) + courseInfoMap + } else { + val name = courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + val category = courseMetadata.getOrElse("primarycategory", "").asInstanceOf[String] + val version = courseMetadata.getOrElse("versionkey", "").asInstanceOf[String] + val parentCollections = courseMetadata + .getOrElse("parentcollections", new java.util.ArrayList()) + .asInstanceOf[java.util.ArrayList[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", name) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap.put("primaryCategory", category) + courseInfoMap.put(config.versionKey, version) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: PostPublishProcessorConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + +} diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala index 0b02f3620..1db1b27d4 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorConfig.scala @@ -6,7 +6,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor import org.apache.flink.streaming.api.scala.OutputTag import org.sunbird.job.BaseJobConfig import org.sunbird.job.postpublish.functions.PublishMetadata - +import scala.collection.immutable.Map import java.util class PostPublishProcessorConfig(override val config: Config) extends BaseJobConfig(config, "post-publish-processor") { @@ -30,6 +30,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val shallowCopyParallelism: Int = config.getInt("task.shallow_copy.parallelism") val linkDialCodeParallelism: Int = config.getInt("task.link_dialcode.parallelism") val batchCreateParallelism: Int = config.getInt("task.batch_create.parallelism") + val postPublishRelationUpdateParallelism: Int = config.getInt("task.post-publish-relation-update.parallelism") // Metric List val totalEventsCount = "total-events-count" @@ -42,6 +43,9 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val dialLinkSuccessCount = "dial-link-success-count" val dialLinkFailedCount = "dial-link-failed-count" val qrImageGeneratorEventCount = "qr-image-event-count" + val postPublishRelationUpdateEventCount = "post-publish-relation-update-count" + val postPublishRelationUpdateSuccessCount = "post-publish-relation-update-success-count" + val postPublishRelationUpdateFailureCount = "post-publish-relation-update-failure-count" // Cassandra Configurations val dbHost: String = config.getString("lms-cassandra.host") @@ -51,6 +55,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val defaultCertTemplateId = config.getString("lms-cassandra.certTemplateId") val sbSystemSettingsTableName = config.getString("lms-cassandra.systemSettingsTable") val batchTableName = config.getString("lms-cassandra.batchTable") + val hierarchyStoreKeySpace = config.getString("lms-cassandra.hierarchyStoreKeySpace") val dialcodeKeyspaceName = config.getString("dialcode-cassandra.keyspace") val dialcodeTableName = config.getString("dialcode-cassandra.imageTable") @@ -65,6 +70,7 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val shallowContentPublishOutTag: OutputTag[PublishMetadata] = OutputTag[PublishMetadata]("shallow-copied-content-publish") val publishEventOutTag: OutputTag[String] = OutputTag[String]("content-publish-request") val generateQRImageOutTag: OutputTag[String] = OutputTag[String]("qr-image-generator-request") + val postPublishRelationUpdateOutTag:OutputTag[String]= OutputTag[String]("post-publish-relation-update") val searchBaseUrl = config.getString("service.search.basePath") val lmsBaseUrl = config.getString("service.lms.basePath") @@ -77,7 +83,29 @@ class PostPublishProcessorConfig(override val config: Config) extends BaseJobCon val reserveDialCodeAPIPath = learningBaseUrl + "/content/v3/dialcode/reserve" val batchAddCertTemplateAPIPath = lmsBaseUrl + "/private/v1/course/batch/cert/template/add" + // QR Image Generator val QRImageGeneratorTopic: String = config.getString("kafka.qrimage.topic") val primaryCategories: util.List[String] = if (config.hasPath("dialcode.linkable.primaryCategory")) config.getStringList("dialcode.linkable.primaryCategory") else util.Arrays.asList("Course") //List[String]("Course") + + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase+ "/content/v3/read/" + + val contentHierarchyTable: String = "content_hierarchy" + val identifier: String = "identifier" + val Hierarchy: String = "hierarchy" + val children: String = "children" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val course: String = "Course" + val parentCollections: String="parentCollections" + + val contentSystemUpdatePath = learningBaseUrl + "/system/v3/content/update/" + val defaultHeaders = Map[String, String] ("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val name: String = "name" + + val contentCacheStore: Int = 0 + + } diff --git a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala index 3d19d7ab9..bd88ec8fe 100644 --- a/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala +++ b/post-publish-processor/src/main/scala/org/sunbird/job/postpublish/task/PostPublishProcessorStreamTask.scala @@ -47,6 +47,9 @@ class PostPublishProcessorStreamTask(config: PostPublishProcessorConfig, kafkaCo .name("dialcode-link-process").uid("dialcode-link-process").setParallelism(config.linkDialCodeParallelism) linkDialCodeStream.getSideOutput(config.generateQRImageOutTag).addSink(kafkaConnector.kafkaStringSink(config.QRImageGeneratorTopic)) + + processStreamTask.getSideOutput(config.postPublishRelationUpdateOutTag).process(new PostPublishRelationUpdaterFunction(config, httpUtil)) + .name("post-publish-relation-update-process").uid("post-publish-relation-update-process").setParallelism(config.postPublishRelationUpdateParallelism) env.execute(config.jobName) } } diff --git a/program-activity-aggregate-updater/pom.xml b/program-activity-aggregate-updater/pom.xml new file mode 100644 index 000000000..e5d97f18a --- /dev/null +++ b/program-activity-aggregate-updater/pom.xml @@ -0,0 +1,244 @@ + + + + 4.0.0 + + org.sunbird + knowledge-platform-jobs + 1.0 + + program-activity-aggregate-updater + 1.0.0 + jar + program-activity-aggregate-updater + + Program Progress Computation + + + + UTF-8 + 1.4.0 + + + + + org.apache.flink + flink-streaming-scala_${scala.version} + ${flink.version} + provided + + + org.sunbird + jobs-core + 1.0.0 + + + joda-time + joda-time + 2.10.6 + + + com.twitter + storehaus-cache_${scala.version} + 0.15.0 + + + org.sunbird + jobs-core + 1.0.0 + test-jar + test + + + org.apache.flink + flink-test-utils_${scala.version} + ${flink.version} + test + + + org.apache.flink + flink-runtime_${scala.version} + ${flink.version} + test + tests + + + it.ozimov + embedded-redis + 0.7.1 + test + + + org.apache.flink + flink-streaming-java_${scala.version} + ${flink.version} + test + tests + + + org.scalatest + scalatest_${scala.version} + 3.0.6 + test + + + org.mockito + mockito-core + 3.3.3 + test + + + com.fiftyonred + mock-jedis + 0.4.0 + test + + + org.cassandraunit + cassandra-unit + 3.11.2.0 + test + + + it.ozimov + embedded-redis + 0.7.1 + test + + + com.google.guava + guava + + + + + + + src/main/scala + src/test/scala + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + + package + + shade + + + + + com.google.code.findbugs:jsr305 + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterStreamTask + + + + reference.conf + + + + + + + + + net.alchim31.maven + scala-maven-plugin + 4.4.0 + + 11 + 11 + ${scala.maj.version} + false + + + + scala-compile-first + process-resources + + add-source + compile + + + + scala-test-compile + process-test-resources + + testCompile + + + + + + + maven-surefire-plugin + 2.22.2 + + true + + + + + org.scalatest + scalatest-maven-plugin + 1.0 + + ${project.build.directory}/surefire-reports + . + dp-duplication-testsuite.txt + + + + test + + test + + + + + + org.scoverage + scoverage-maven-plugin + ${scoverage.plugin.version} + + ${scala.version} + true + true + + + + + + \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/resources/log4j.properties b/program-activity-aggregate-updater/src/main/resources/log4j.properties new file mode 100644 index 000000000..cacd49c79 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/resources/log4j.properties @@ -0,0 +1,11 @@ +# log4j.appender.file=org.apache.log4j.FileAppender +log4j.appender.file=org.apache.log4j.RollingFileAppender +log4j.appender.file.file=course-metrics-updater.log +log4j.appender.file.append=true +log4j.appender.file.layout=org.apache.log4j.PatternLayout +log4j.appender.file.MaxFileSize=256KB +log4j.appender.file.MaxBackupIndex=4 +log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + +# Suppress the irrelevant (wrong) warnings from the Netty channel handler +log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf new file mode 100644 index 000000000..a51a37d1d --- /dev/null +++ b/program-activity-aggregate-updater/src/main/resources/program-activity-aggregate-updater.conf @@ -0,0 +1,53 @@ +include "base-config.conf" + +kafka { + input.topic = "sunbirddev.coursebatch.job.request" + output.audit.topic = "sunbirddev.telemetry.raw" + output.failed.topic = "sunbirddev.activity.agg.failed" + output.certissue.topic = "sunbirddev.issue.certificate.request" + groupId = "sunbirddev-program-activity-aggregate-updater-group" +} + +task { + window.shards = 1 + consumer.parallelism = 1 + dedup.parallelism = 1 + activity.agg.parallelism = 1 + enrolment.complete.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + consumption.table = "user_content_consumption" + user_activity_agg.table = "user_activity_agg" + user_enrolments.table = "user_enrolments" +} + +redis { + database { + relationCache.id = 10 + } +} + +dedup-redis { + host = 11.2.4.22 + port = 6379 + database.index = 3 + database.expiry = 604800 +} + +threshold.batch.read.interval = 60 // In sec +threshold.batch.read.size = 1000 +threshold.batch.write.size = 10 + +activity { + module.aggs.enabled = true + input.dedup.enabled = true + filter.processed.enrolments = true + collection.status.cache.expiry = 3600 +} + +service { + content.basePath = "http://11.2.6.6/content" + search.basePath = "http://11.2.6.6/search" +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala new file mode 100644 index 000000000..fda596c54 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/common/DeDupHelper.scala @@ -0,0 +1,12 @@ +package org.sunbird.job.programaggregate.common + +import java.security.MessageDigest + +object DeDupHelper { + + def getMessageId(collectionId: String, batchId: String, userId: String, contentId: String, status: Int): String = { + val key = Array(collectionId, batchId, userId, contentId, status).mkString("|") + MessageDigest.getInstance("MD5").digest(key.getBytes).map("%02X".format(_)).mkString; + } + +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala new file mode 100644 index 000000000..c694e5665 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/domain/Models.scala @@ -0,0 +1,52 @@ +package org.sunbird.job.programaggregate.domain + +import java.util +import java.util.{Date, UUID} + +import scala.collection.JavaConverters._ + + +case class ActorObject(id: String, `type`: String = "User") + +case class EventContext(channel: String = "in.sunbird", + env: String = "Course", + sid: String = UUID.randomUUID().toString, + did: String = UUID.randomUUID().toString, + pdata: util.Map[String, String] = Map("ver" -> "3.0", "id" -> "org.sunbird.learning.platform", "pid" -> "course-progress-updater").asJava, + cdata: Array[util.Map[String, String]]) + + +case class EventData(props: Array[String], `type`: String) + +case class EventObject(id: String, `type`: String, rollup: util.Map[String, String]) + +case class TelemetryEvent(actor: ActorObject, + eid: String = "AUDIT", + edata: EventData, + ver: String = "3.0", + syncts: Long = System.currentTimeMillis(), + ets: Long = System.currentTimeMillis(), + context: EventContext = EventContext( + cdata = Array[util.Map[String, String]]() + ), + mid: String = s"LP.AUDIT.${UUID.randomUUID().toString}", + `object`: EventObject, + tags: util.List[AnyRef] = new util.ArrayList[AnyRef]() + ) + +case class ContentStatus(contentId: String, status: Int = 0, completedCount: Int = 0, viewCount: Int = 1, fromInput: Boolean = true, eventsFor: List[String] = List()) + +case class UserContentConsumption(userId: String, batchId: String, courseId: String, contents: Map[String, ContentStatus]) + +case class UserActivityAgg(activity_type: String, + user_id: String, + activity_id: String, + context_id: String, + aggregates: Map[String, Double], + agg_last_updated: Map[String, Long] + ) + +case class CollectionProgress(userId: String, batchId: String, courseId: String, progress: Int, completedOn: Date, contentStatus: Map[String, Int], inputContents: List[String], completed: Boolean = false) + +case class UserEnrolmentAgg(activityAgg: UserActivityAgg, collectionProgress: Option[CollectionProgress] = None) + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala new file mode 100644 index 000000000..ea720f482 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesEnrolUpdateFunction.scala @@ -0,0 +1,268 @@ +package org.sunbird.job.programaggregate.functions + +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} +import com.google.common.reflect.TypeToken +import com.google.gson.Gson +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil} +import org.sunbird.job.{Metrics, WindowBaseProcessFunction} + +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions.`seq AsJavaList` +import scala.collection.mutable.ListBuffer + + + +class ProgramActivityAggregatesEnrolUpdateFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) + (implicit val stringTypeInfo: TypeInformation[String]) + extends WindowBaseProcessFunction[Map[String, AnyRef], String, Int](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramActivityAggregatesEnrolUpdateFunction]) + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() + + override def metricsList(): List[String] = { + List(config.failedEventCount, config.dbUpdateCount, config.dbReadCount, config.cacheHitCount, config.cacheMissCount, config.processedEnrolmentCount, config.retiredCCEventsCount) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) + } + + override def close(): Unit = { + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } + super.close() + } + + override def process(key: Int, + context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, + events: Iterable[Map[String, AnyRef]], + metrics: Metrics): Unit = { + logger.info("Event Info Inside ProgramActivityAggregrator: " + events) + val inputUserConsumptionList: List[UserContentConsumption] = events + .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) + .values.map(value => { + metrics.incCounter(config.processedEnrolmentCount) + val batchId = value.head(config.batchId).toString + val userId = value.head(config.userId).toString + val courseId = value.head(config.courseId).toString + logger.info("courseId: " + courseId + " batchId: " + batchId) + val userConsumedContents = value.head(config.contents).asInstanceOf[List[Map[String, AnyRef]]] + val enrichedContents = getContentStatusFromEvent(userConsumedContents) + UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) + }).toList + logger.info("the input user ConsumptionList:" + inputUserConsumptionList) + if (inputUserConsumptionList.isEmpty) + return + + val updateProgramEnrollments = updateProgramEnrollment(inputUserConsumptionList)(metrics) + + val collectionProgressUpdateList = updateProgramEnrollments.filter(progress => !progress.completed) + context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) + + logger.info("collectionProgressUpdateList Queries List :" + collectionProgressUpdateList) + val collectionProgressCompleteList = updateProgramEnrollments.filter(progress => progress.completed) + context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) + + logger.info("collectionProgressCompleteList Queries List :" + collectionProgressCompleteList) + } + + def getContentStatusFromEvent(contents: List[Map[String, AnyRef]]): Map[String, ContentStatus] = { + val enrichedContents = contents.map(content => { + (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) + }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) + .map(x => { + val completedCount = if (x._2.intValue() == 2) 1 else 0 + ContentStatus(x._1, x._2.intValue(), completedCount) + }).groupBy(f => f.contentId) + + enrichedContents.map(content => { + val consumedList = content._2 + val finalStatus = consumedList.map(x => x.status).max + val views = sumFunc(consumedList, (x: ContentStatus) => { + x.viewCount + }) + val completion = sumFunc(consumedList, (x: ContentStatus) => { + x.completedCount + }) + (content._1, ContentStatus(content._1, finalStatus, completion, views)) + }) + } + + /** + * Computation of Sum for viewCount and completedCount. + */ + private def sumFunc(list: List[ContentStatus], valFunc: ContentStatus => Int): Int = list.map(x => valFunc(x)).sum + + def updateProgramEnrollment(events: List[UserContentConsumption])(implicit metrics: Metrics): List[CollectionProgress] = { + + val contentEnrollProgress: List[CollectionProgress] = events.flatMap { collectionProgress => + val updatedEnrolContentConsumption = updateEnrolContentConsumption(collectionProgress)(metrics) + if (updatedEnrolContentConsumption != null) + programEnrolConsumption(updatedEnrolContentConsumption)(metrics) + else + None + } + contentEnrollProgress + } + + def getEnrolment(userId: String, programId: String)(implicit metrics: Metrics): Row = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq(config.userId, userId)) + .and(QueryBuilder.eq(config.courseId, programId)) + metrics.incCounter(config.dbReadCount) + var row: java.util.List[Row] = cassandraUtil.find(selectWhere.toString) + if (null != row) { + if (row.size() == 1) { + row.asScala.get(0) + } else { + logger.error("Enrollement is more than 1, for programId:" + programId + " userId:" + userId) + null + } + } else { + logger.error("No Enrollement found for programId: " + programId + " userId: " + userId) + null + } + } + + def readFromCache(key: String, metrics: Metrics): List[String] = { + metrics.incCounter(config.cacheHitCount) + val list = cache.getKeyMembers(key) + if (CollectionUtils.isEmpty(list)) { + metrics.incCounter(config.cacheMissCount) + logger.info("Redis cache (smembers) not available for key: " + key) + } + list.asScala.toList + } + + def updateEnrolContentConsumption(userConsumption: UserContentConsumption)(implicit metrics: Metrics): UserContentConsumption = { + val programEnrollmentStatus = getEnrolment(userConsumption.userId, userConsumption.courseId)(metrics) + if (programEnrollmentStatus != null && programEnrollmentStatus.getInt("status") != 2) { + val programContentStatusList = ListBuffer[Map[String, AnyRef]]() + val programContentStatus = Option(programEnrollmentStatus.getMap( + config.contentStatus, TypeToken.of(classOf[String]), TypeToken.of(classOf[Integer]))).head + for ((key, value) <- userConsumption.contents) { + // Check if the key is present in leafNodeMap + if (programContentStatus.get(key) != null) { + if (value.status == 2 && programContentStatus.get(key) != 2) { + // Update progress in contentStatus for the matching key + programContentStatus.put(key, value.status) + } + } else { + programContentStatus.put(key, value.status) + } + } + + for( (key, value) <- programContentStatus.asScala) { + val contentStatusMap = Map( + config.contentId -> key, + config.status -> value + ) + programContentStatusList += contentStatusMap + } + // Add programContentStatusMap to programContentStatusList + val updatedContent = getContentStatusFromEvent(programContentStatusList.toList) + val updatedUserConsumption = userConsumption.copy(contents = updatedContent) + updatedUserConsumption + } else { + null + } + } + + def programEnrolConsumption(userConsumption: UserContentConsumption)(implicit metrics: Metrics): Option[CollectionProgress] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + val key = s"$courseId:$courseId:${config.leafNodes}" + val leafNodes = readFromCache(key, metrics).distinct + if (leafNodes.isEmpty) { + logger.error(s"leaf nodes are not available for: $key") + //context.output(config.failedEventOutputTag, gson.toJson(userConsumption)) + val status = getCollectionStatus(courseId) + if (StringUtils.equals("Retired", status)) { + metrics.incCounter(config.retiredCCEventsCount) + println(s"contents consumed from a retired collection: $courseId") + logger.warn(s"contents consumed from a retired collection: $courseId") + None + } else { + metrics.incCounter(config.failedEventCount) + val message = s"leaf nodes are not available for a published collection: $courseId" + logger.error(message) + throw new Exception(message) + } + } else { + val completedCount = leafNodes.intersect(userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct).size + val contentStatus = userConsumption.contents.map(cc => (cc._2.contentId, cc._2.status)).toMap + val inputContents = userConsumption.contents.filter(cc => cc._2.fromInput).keys.toList + val collectionProgress = if (completedCount >= leafNodes.size) { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, new java.util.Date(), contentStatus, inputContents, true)) + } else { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, null, contentStatus, inputContents)) + } + return collectionProgress + } + } + + def getCollectionStatus(collectionId: String): String = { + val cacheStatus = collectionStatusCache.getNonExpired(collectionId).getOrElse("") + if (StringUtils.isEmpty(cacheStatus)) { + val dbStatus = getDBStatus(collectionId) + collectionStatusCache = collectionStatusCache.putClocked(collectionId, dbStatus)._2 + dbStatus + } else cacheStatus + } + + def getDBStatus(collectionId: String): String = { + val requestBody = + s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "$collectionId", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + val response = httpUtil.post(config.searchAPIURL, requestBody) + if (response.status == 200) { + val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) + val result = responseBody.getOrDefault("result", new java.util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] + val count = result.getOrDefault("count", 0.asInstanceOf[Number]).asInstanceOf[Number].intValue() + if (count > 0) { + val list = result.getOrDefault("content", new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + list.asScala.head.get("status").asInstanceOf[String] + } else throw new Exception(s"There are no published or retired collection with id: $collectionId") + } else { + logger.error("search-service error: " + response.body) + throw new Exception("search-service not returning error:" + response.status) + } + } +} + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala new file mode 100644 index 000000000..ebc0c462f --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramActivityAggregatesFunction.scala @@ -0,0 +1,639 @@ +package org.sunbird.job.programaggregate.functions + +import java.lang.reflect.Type +import java.util.concurrent.TimeUnit +import com.datastax.driver.core.Row +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.scala.function.ProcessWindowFunction +import org.apache.flink.streaming.api.windowing.windows.GlobalWindow +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.{Metrics, WindowBaseProcessFunction} + +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class ProgramActivityAggregatesFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null) + (implicit val stringTypeInfo: TypeInformation[String]) + extends WindowBaseProcessFunction[Map[String, AnyRef], String, Int](config) { + + val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramActivityAggregatesFunction]) + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() + + override def metricsList(): List[String] = { + List(config.failedEventCount, config.dbUpdateCount, config.dbReadCount, config.cacheHitCount, config.cacheMissCount, config.processedEnrolmentCount, config.retiredCCEventsCount) + } + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) + } + + override def close(): Unit = { + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } + super.close() + } + + override def process(key: Int, + context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context, + events: Iterable[Map[String, AnyRef]], + metrics: Metrics): Unit = { + logger.info("Event Info Inside ProgramActivityAggregrator: " + events) + val inputUserConsumptionList: List[UserContentConsumption] = events + .groupBy(key => (key.get(config.courseId), key.get(config.batchId), key.get(config.userId))) + .values.map(value => { + metrics.incCounter(config.processedEnrolmentCount) + val batchId = value.head(config.batchId).toString + val userId = value.head(config.userId).toString + val courseId = value.head(config.courseId).toString + logger.info("courseId: " + courseId + " batchId: " + batchId) + val userConsumedContents = value.head(config.contents).asInstanceOf[List[Map[String, AnyRef]]] + val enrichedContents = getContentStatusFromEvent(userConsumedContents) + UserContentConsumption(userId = userId, batchId = batchId, courseId = courseId, enrichedContents) + }).toList + logger.info("the input user ConsumptionList:" + inputUserConsumptionList) + if (inputUserConsumptionList.isEmpty) + return + // Fetch the content status from the table in batch format + val dbUserConsumption: Map[String, UserContentConsumption] = getContentStatusFromDB(events.toList, metrics) + + logger.info("Content Consumption List :" + dbUserConsumption) + // Final User's ContentConsumption after merging with DB data. + // Here we have final viewcount, completedcount and identified the content which should generate AUDIT events for start and complete. + val finalUserConsumptionList = inputUserConsumptionList.map(inputData => { + val dbData = dbUserConsumption.getOrElse(getUCKey(inputData), UserContentConsumption(inputData.userId, inputData.batchId, inputData.courseId, Map())) + finalUserConsumption(inputData, dbData)(metrics) + }) + + logger.info("finalUserConsumptionList List :" + finalUserConsumptionList) + // user_content_consumption update with viewcount and completedcout. + val userConsumptionQueries = finalUserConsumptionList.flatMap(userConsumption => getContentConsumptionQueries(userConsumption)) + updateDB(config.thresholdBatchWriteSize, userConsumptionQueries)(metrics) + + logger.info("UserConsumption Queries List :" + userConsumptionQueries) + + val courseAggregations = finalUserConsumptionList.flatMap(userConsumption => { + + // Course Level Agg using the merged data of ContentConsumption per user, course and batch. + val optCourseAgg = courseActivityAgg(userConsumption, context)(metrics) + val courseAggs = if (optCourseAgg.nonEmpty) List(optCourseAgg.get) else List() + + // Identify the children of the course (only collections) for which aggregates computation required. + // Computation of aggregates using leafNodes (of the specific collection) and user completed contents. + // Here computing only "completedCount" aggregate. + if (config.moduleAggEnabled) { + val courseChildrenAggs = courseChildrenActivityAgg(userConsumption)(metrics) + courseAggs ++ courseChildrenAggs + } else courseAggs + }) + + logger.info("courseAggregations Queries List :" + courseAggregations) + // Saving all queries for course and it's children (only collection) aggregates. + val aggQueries = courseAggregations.map(agg => getUserAggQuery(agg.activityAgg)) + updateDB(config.thresholdBatchWriteSize, aggQueries)(metrics) + + // Saving enrolment completion data. + val collectionProgressList = courseAggregations.filter(agg => agg.collectionProgress.nonEmpty).map(agg => agg.collectionProgress.get) + + val collectionProgressUpdateList = collectionProgressList.filter(progress => !progress.completed) + context.output(config.collectionUpdateOutputTag, collectionProgressUpdateList) + + logger.info("collectionProgressUpdateList Queries List :" + collectionProgressUpdateList) + val collectionProgressCompleteList = collectionProgressList.filter(progress => progress.completed) + context.output(config.collectionCompleteOutputTag, collectionProgressCompleteList) + + logger.info("collectionProgressCompleteList Queries List :" + collectionProgressCompleteList) + // Content AUDIT Event generation and pushing to output tag. + finalUserConsumptionList.flatMap(userConsumption => contentAuditEvents(userConsumption)).foreach(event => context.output(config.auditEventOutputTag, gson.toJson(event))) + } + + /** + * Course Level Agg using the merged data of ContentConsumption per user, course and batch. + */ + def courseActivityAgg(userConsumption: UserContentConsumption, context: ProcessWindowFunction[Map[String, AnyRef], String, Int, GlobalWindow]#Context)(implicit metrics: Metrics): Option[UserEnrolmentAgg] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + val key = s"$courseId:$courseId:${config.leafNodes}" + val leafNodes = readFromCache(key, metrics).distinct + if (leafNodes.isEmpty) { + logger.error(s"leaf nodes are not available for: $key") + context.output(config.failedEventOutputTag, gson.toJson(userConsumption)) + val status = getCollectionStatus(courseId) + if (StringUtils.equals("Retired", status)) { + metrics.incCounter(config.retiredCCEventsCount) + println(s"contents consumed from a retired collection: $courseId") + logger.warn(s"contents consumed from a retired collection: $courseId") + None + } else { + metrics.incCounter(config.failedEventCount) + val message = s"leaf nodes are not available for a published collection: $courseId" + logger.error(message) + throw new Exception(message) + } + } else { + val completedCount = leafNodes.intersect(userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct).size + val contentStatus = userConsumption.contents.map(cc => (cc._2.contentId, cc._2.status)).toMap + val inputContents = userConsumption.contents.filter(cc => cc._2.fromInput).keys.toList + val collectionProgress = if (completedCount >= leafNodes.size) { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, new java.util.Date(), contentStatus, inputContents, true)) + } else { + Option(CollectionProgress(userId, userConsumption.batchId, courseId, completedCount, null, contentStatus, inputContents)) + } + Option(UserEnrolmentAgg(UserActivityAgg("Course", userId, courseId, contextId, Map("completedCount" -> completedCount.toDouble), Map("completedCount" -> System.currentTimeMillis())), collectionProgress)) + } + } + + /** + * Identified the children of the course (only collections) for which aggregates computation required. + * Computation of aggregates using leafNodes (of the specific collection) and user completed contents. + * Here computing only "completedCount" aggregate. + */ + def courseChildrenActivityAgg(userConsumption: UserContentConsumption)(implicit metrics: Metrics): List[UserEnrolmentAgg] = { + val courseId = userConsumption.courseId + val userId = userConsumption.userId + val contextId = "cb:" + userConsumption.batchId + + // These are the child collections which require computation of aggregates - for this user. + val ancestors = userConsumption.contents.mapValues(content => { + val contentId = content.contentId + readFromCache(key = s"$courseId:$contentId:${config.ancestors}", metrics) + }).values.flatten.filter(a => !StringUtils.equals(a, courseId)).toList.distinct + + // LeafNodes of the identified child collections - for this user. + val collectionsWithLeafNodes = ancestors.map(unitId => { + (unitId, readFromCache(key = s"$courseId:$unitId:${config.leafNodes}", metrics).distinct) + }).toMap + + // Content completed - By this user. + val userCompletedContents = userConsumption.contents.filter(cc => cc._2.status == 2).map(cc => cc._2.contentId).toList.distinct + + // Child Collection UserAggregate list - for this user. + collectionsWithLeafNodes.map(e => { + val collectionId = e._1 + val leafNodes = e._2 + val completedCount = leafNodes.intersect(userCompletedContents).size + /* TODO - List + TODO 1. Generalise activityType from "Course" to "Collection". + TODO 2.Identify how to generate start and end event for CourseUnit. + */ + val activityAgg = UserActivityAgg("Course", userId, collectionId, contextId, Map("completedCount" -> completedCount), Map("completedCount" -> System.currentTimeMillis())) + UserEnrolmentAgg(activityAgg, None) + }).toList + } + + /** + * Generation of a "String" key for UserContentConsumption. + */ + def getUCKey(userConsumption: UserContentConsumption): String = { + userConsumption.userId + ":" + userConsumption.courseId + ":" + userConsumption.batchId + } + + /** + * Merging the Input and DB ContentStatus data of a User, Course and Batch (Enrolment) + * This is the critical part of the code. + */ + def finalUserConsumption(inputData: UserContentConsumption, dbData: UserContentConsumption)(implicit metrics: Metrics): UserContentConsumption = { + val dbContents = dbData.contents + val processedContents = inputData.contents.map { + case (contentId, inputCC) => { + // ContentStatus from DB. + val dbCC: ContentStatus = dbContents.getOrElse(contentId, ContentStatus(contentId, 0, 0, 0)) + val finalStatus = List(inputCC.status, dbCC.status).max // Final status is max of DB and Input ContentStatus. + val views = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { + x.viewCount + }) // View Count is sum of DB and Input ContentStatus. + val completion = sumFunc(List(inputCC, dbCC), (x: ContentStatus) => { + x.completedCount + }) // Completed Count is sum of DB and Input ContentStatus. + val eventsFor: List[String] = getEventActions(dbCC, inputCC) + // Merged ContentStatus. + (contentId, ContentStatus(contentId, finalStatus, completion, views, inputCC.fromInput, eventsFor)) + } + } + + val existingContents = processedContents.keys.toList + val remainingContents = dbData.contents.filterKeys(key => !existingContents.contains(key)) + val finalContentsMap = processedContents ++ remainingContents + UserContentConsumption(inputData.userId, inputData.batchId, inputData.courseId, finalContentsMap) + } + + /** + * This will identify whether this is the start or complete of the Content by User. + * + * @return List - Actions - "start" and "complete". + */ + def getEventActions(dbCC: ContentStatus, inputCC: ContentStatus): List[String] = { + val startAction = if (dbCC.viewCount == 0) List("start") else List() + val completeAction = if (dbCC.completedCount == 0 && inputCC.completedCount > 0) List(config.complete) else List() + startAction ::: completeAction + } + + /** + * Generic method to read data from DB (Cassandra). + * + * @return + */ + def readFromDB(columns: Map[String, AnyRef], keySpace: String, table: String, metrics: Metrics): List[Row] = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(keySpace, table). + where() + columns.map(col => { + col._2 match { + case value: List[Any] => + selectWhere.and(QueryBuilder.in(col._1, value.asJava)) + case _ => + selectWhere.and(QueryBuilder.eq(col._1, col._2)) + } + }) + metrics.incCounter(config.dbReadCount) + cassandraUtil.find(selectWhere.toString).asScala.toList + + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + val result = cassandraUtil.upsert(cqlBatch.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed: " + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } + + def readFromCache(key: String, metrics: Metrics): List[String] = { + metrics.incCounter(config.cacheHitCount) + val list = cache.getKeyMembers(key) + if (CollectionUtils.isEmpty(list)) { + metrics.incCounter(config.cacheMissCount) + logger.info("Redis cache (smembers) not available for key: " + key) + } + list.asScala.toList + } + + def getUserAggQuery(progress: UserActivityAgg): + Update.Where = { + QueryBuilder.update(config.dbKeyspace, config.dbUserActivityAggTable) + .`with`(QueryBuilder.putAll(config.aggregates, progress.aggregates.asJava)) + .and(QueryBuilder.putAll(config.aggLastUpdated, progress.agg_last_updated.asJava)) + .where(QueryBuilder.eq(config.activityId, progress.activity_id)) + .and(QueryBuilder.eq(config.activityType, progress.activity_type)) + .and(QueryBuilder.eq(config.contextId, progress.context_id)) + .and(QueryBuilder.eq(config.activityUser, progress.user_id)) + } + + /** + * Creates the cql query for content consumption table + */ + def getContentConsumptionQueries(userContentConsumption: UserContentConsumption): List[Update.Where] = { + userContentConsumption.contents.mapValues(content => { + QueryBuilder.update(config.dbKeyspace, config.dbUserContentConsumptionTable) + .`with`(QueryBuilder.set(config.viewcount, content.viewCount)) + .and(QueryBuilder.set(config.completedcount, content.completedCount)) + .where(QueryBuilder.eq(config.batchId.toLowerCase(), userContentConsumption.batchId)) + .and(QueryBuilder.eq(config.courseId.toLowerCase(), userContentConsumption.courseId)) + .and(QueryBuilder.eq(config.userId.toLowerCase(), userContentConsumption.userId)) + .and(QueryBuilder.eq(config.contentId.toLowerCase(), content.contentId)) + }).values.toList + } + + /** + * Method to get the content status object in map format ex: (do_5874308329084 -> 2, do_59485345435 -> 3) + * It always takes the highest precedence progress values for the contents ex: (do_5874308329084 -> 2, do_5874308329084 -> 1, do_59485345435 -> 3) => (do_5874308329084 -> 2, do_59485345435 -> 3) + * + * Ex: Map("C1"->2, "C2" ->1) + * + */ + def getContentStatusFromEvent(contents: List[Map[String, AnyRef]]): Map[String, ContentStatus] = { + val enrichedContents = contents.map(content => { + (content.getOrElse(config.contentId, "").asInstanceOf[String], content.getOrElse(config.status, 0).asInstanceOf[Number]) + }).filter(t => StringUtils.isNotBlank(t._1) && (t._2.intValue() > 0)) + .map(x => { + val completedCount = if (x._2.intValue() == 2) 1 else 0 + ContentStatus(x._1, x._2.intValue(), completedCount) + }).groupBy(f => f.contentId) + + enrichedContents.map(content => { + val consumedList = content._2 + val finalStatus = consumedList.map(x => x.status).max + val views = sumFunc(consumedList, (x: ContentStatus) => { + x.viewCount + }) + val completion = sumFunc(consumedList, (x: ContentStatus) => { + x.completedCount + }) + (content._1, ContentStatus(content._1, finalStatus, completion, views)) + }) + } + + /** + * Computation of Sum for viewCount and completedCount. + */ + private def sumFunc(list: List[ContentStatus], valFunc: ContentStatus => Int): Int = list.map(x => valFunc(x)).sum + + + /** + * Method to get the content status from the database + * + * Ex: List(Map("courseId" -> "do_43795", batchId -> "batch1", userId->"user001", contentStatus -> Map("C1"->2, "C2" ->1))) + * + */ + def getContentStatusFromDB(eDataBatch: List[Map[String, AnyRef]], metrics: Metrics): Map[String, UserContentConsumption] = { + + val contentConsumption = scala.collection.mutable.Map[String, UserContentConsumption]() + val primaryFields = Map( + config.userId.toLowerCase() -> eDataBatch.map(x => x(config.userId)).distinct, + config.batchId.toLowerCase -> eDataBatch.map(x => x(config.batchId)).distinct, + config.courseId.toLowerCase -> eDataBatch.map(x => x(config.courseId)).distinct + ) + + val records = Option(readFromDB(primaryFields, config.dbKeyspace, config.dbUserContentConsumptionTable, metrics)) + records.map(record => record.groupBy(col => Map(config.batchId -> col.getObject(config.batchId.toLowerCase()).asInstanceOf[String], config.userId -> col.getObject(config.userId.toLowerCase()).asInstanceOf[String], config.courseId -> col.getObject(config.courseId.toLowerCase()).asInstanceOf[String]))) + .foreach(groupedRecords => groupedRecords.map(entry => { + val identifierMap = entry._1 + val consumptionList = entry._2.flatMap(row => Map(row.getObject(config.contentId.toLowerCase()).asInstanceOf[String] -> Map(config.status -> row.getObject(config.status), config.viewcount -> row.getObject(config.viewcount), config.completedcount -> row.getObject(config.completedcount)))) + .map(entry => { + val contentStatus = entry._2.filter(x => x._2 != null) + val contentId = entry._1 + val status = contentStatus.getOrElse(config.status, 1).asInstanceOf[Number].intValue() + val viewCount = contentStatus.getOrElse(config.viewcount, 0).asInstanceOf[Number].intValue() + val completedCount = contentStatus.getOrElse(config.completedcount, 0).asInstanceOf[Number].intValue() + (contentId, ContentStatus(contentId, status, completedCount, viewCount, false)) + }).toMap + + val userId = identifierMap(config.userId) + val batchId = identifierMap(config.batchId) + val courseId = identifierMap(config.courseId) + + val userContentConsumption = UserContentConsumption(userId, batchId, courseId, consumptionList) + contentConsumption += getUCKey(userContentConsumption) -> userContentConsumption + + })) + contentConsumption.toMap + } + + /** + * Content - AUDIT Event Generation using UserContentConsumption + * "eventsFor" - will have the action (or type) for the event to generate. + */ + def contentAuditEvents(userConsumption: UserContentConsumption): List[TelemetryEvent] = { + val userId = userConsumption.userId + val courseId = userConsumption.courseId + val batchId = userConsumption.batchId + val contentsForEvents = userConsumption.contents.filter(c => c._2.eventsFor.nonEmpty).values + contentsForEvents.flatMap(c => { + c.eventsFor.map(action => { + val properties = if (StringUtils.equalsIgnoreCase(action, config.complete)) Array(config.viewcount, config.completedcount) else Array(config.viewcount) + TelemetryEvent( + actor = ActorObject(id = userId), + edata = EventData(props = properties, `type` = action), // action values are "start", "complete". + context = EventContext(cdata = Array(Map("type" -> config.courseBatch, "id" -> batchId).asJava)), + `object` = EventObject(id = c.contentId, `type` = "Content", rollup = Map[String, String]("l1" -> courseId).asJava) + ) + }) + }).toList + } + + def getDBStatus(collectionId: String): String = { + val requestBody = + s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "$collectionId", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + val response = httpUtil.post(config.searchAPIURL, requestBody) + if (response.status == 200) { + val responseBody = gson.fromJson(response.body, classOf[java.util.Map[String, AnyRef]]) + val result = responseBody.getOrDefault("result", new java.util.HashMap[String, AnyRef]()).asInstanceOf[java.util.Map[String, AnyRef]] + val count = result.getOrDefault("count", 0.asInstanceOf[Number]).asInstanceOf[Number].intValue() + if (count > 0) { + val list = result.getOrDefault("content", new java.util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[java.util.List[java.util.Map[String, AnyRef]]] + list.asScala.head.get("status").asInstanceOf[String] + } else throw new Exception(s"There are no published or retired collection with id: $collectionId") + } else { + logger.error("search-service error: " + response.body) + throw new Exception("search-service not returning error:" + response.status) + } + } + + def getCollectionStatus(collectionId: String): String = { + val cacheStatus = collectionStatusCache.getNonExpired(collectionId).getOrElse("") + if (StringUtils.isEmpty(cacheStatus)) { + val dbStatus = getDBStatus(collectionId) + collectionStatusCache = collectionStatusCache.putClocked(collectionId, dbStatus)._2 + dbStatus + } else cacheStatus + } + +/* def verifyPrimaryCategory(identifier: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): Boolean = { + logger.info( + "Verify Program post-publish required for content: " + identifier + ) + // Get the primary Categories for the courses here + var isValidProgram = false + val contentObj: java.util.Map[String, AnyRef] = + getCourseInfo(identifier)(metrics, config, cache, httpUtil) + if (!contentObj.isEmpty) { + val primaryCategory = contentObj.get("primaryCategory") + if (primaryCategory != null && + (primaryCategory == "Program" + || primaryCategory == "Curated Program" + || primaryCategory == "Blended Program")) { + isValidProgram = true + } + logger.info("PrimaryCategory value is :" + primaryCategory + ", for Id: " + identifier) + } else { + logger.error("Failed to read content details for Id: " + identifier) + } + logger.info("is program activity aggregator is skipping this event ? " + isValidProgram) + isValidProgram + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory,parentCollections" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + + def getProgramEvent(eventData: Map[String, AnyRef])( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): mutable.Iterable[Map[String, AnyRef]] = { + var eventInfoMap: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] + val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] + val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(courseId)(metrics, config, cache, httpUtil) + val primaryCategory: String = contentObj.get(config.primaryCategory).asInstanceOf[String] + val parentCollections: List[String] = contentObj.get(config.parentCollections).asInstanceOf[List[String]] + logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) + if (config.validProgramPrimaryCategory.contains(primaryCategory)) { + eventInfoMap += eventData + var eventInfo: Map[String, AnyRef] = Map.empty + eventInfo ++= eventData + if (StringUtils.isEmpty(batchId)) { + val row = getEnrolment(userId, courseId)(metrics) + if (row != null) { + eventInfo += ("batchId" -> row.getString("batchid")) + } else { + return null; + } + eventInfoMap += eventInfo + } + } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) + && !parentCollections.isEmpty) { + for (parentId <- parentCollections) { + val row = getEnrolment(userId, parentId)(metrics) + if (row != null) { + val contentConsumption = eventData.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] + val eventInfoProgram = Map[String, AnyRef]("contents" -> contentConsumption, + "userId" -> userId, + "action" -> "batch-enrolment-update", + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) + eventInfoMap += eventInfoProgram + logger.info("EventMapInfoProgram:" + eventInfoProgram) + } + } + } else { + logger.error("Not Valid Primary Category: " + primaryCategory + " parentCollections: " + parentCollections) + } + eventInfoMap + } + + def getEnrolment(userId: String, courseId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + }*/ +} + diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala new file mode 100644 index 000000000..c09d32901 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramContentConsumptionDeDupFunction.scala @@ -0,0 +1,241 @@ +package org.sunbird.job.programaggregate.functions + +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select} +import com.google.gson.Gson +import com.google.gson.reflect.TypeToken +import com.twitter.storehaus.cache.TTLCache +import com.twitter.util.Duration +import org.apache.commons.lang3.StringUtils +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.{DataCache, RedisConnect} +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.programaggregate.common.DeDupHelper +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.{CassandraUtil, HttpUtil, ScalaJsonUtil} +import org.sunbird.job.{BaseProcessFunction, Metrics} + +import java.lang.reflect.Type +import java.util +import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ +import scala.collection.mutable + +class ProgramContentConsumptionDeDupFunction(config: ProgramActivityAggregateUpdaterConfig, httpUtil: HttpUtil, @transient var cassandraUtil: CassandraUtil = null)(implicit val stringTypeInfo: TypeInformation[String]) extends BaseProcessFunction[util.Map[String, AnyRef], String](config) { + + val mapType: Type = new TypeToken[Map[String, AnyRef]]() {}.getType + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramContentConsumptionDeDupFunction]) + var deDupEngine: DeDupEngine = _ + private var cache: DataCache = _ + private var collectionStatusCache: TTLCache[String, String] = _ + lazy private val gson = new Gson() + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + cache = new DataCache(config, new RedisConnect(config), config.nodeStore, List()) + cache.init() + collectionStatusCache = TTLCache[String, String](Duration.apply(config.statusCacheExpirySec, TimeUnit.SECONDS)) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + if (cassandraUtil != null) { + cassandraUtil.close() + } + if (cache != null) { + cache.close() + } + deDupEngine.close() + super.close() + } + + override def processElement(event: util.Map[String, AnyRef], context: ProcessFunction[util.Map[String, AnyRef], String]#Context, metrics: Metrics): Unit = { + metrics.incCounter(config.totalEventCount) + val eData = event.get(config.eData).asInstanceOf[util.Map[String, AnyRef]].asScala + val isBatchEnrollmentEvent: Boolean = StringUtils.equalsIgnoreCase(eData.getOrElse(config.action, "").asInstanceOf[String], config.batchEnrolmentUpdateCode) + if (isBatchEnrollmentEvent) { + val contents = eData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala + logger.info("Input Event: " + contents) + var updatedEventInfo: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] + var eventInfoMap: mutable.Iterable[Map[String, AnyRef]] = getProgramEvent(eData.toMap)(metrics, config, httpUtil, cache) + logger.info("EventInfoMap: " + eventInfoMap) + if (eventInfoMap.nonEmpty) { + updatedEventInfo ++= eventInfoMap + } + + logger.info("UpdatedEventInfoMap: " + updatedEventInfo) + + updatedEventInfo.filter(e => discardDuplicates(e)).foreach(d => context.output(config.uniqueConsumptionOutput, d)) + } else metrics.incCounter(config.skipEventsCount) + } + + override def metricsList(): List[String] = { + List(config.totalEventCount, config.skipEventsCount, config.batchEnrolmentUpdateEventCount, config.dbReadCount) + } + + def discardDuplicates(event: Map[String, AnyRef]): Boolean = { + if (config.dedupEnabled) { + val userId = event.getOrElse(config.userId, "").asInstanceOf[String] + val courseId = event.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId = event.getOrElse(config.batchId, "").asInstanceOf[String] + logger.info("Event List inside discardDuplicates" + event) + val contents = event.getOrElse(config.contents, List[Map[String,AnyRef]]()).asInstanceOf[List[Map[String, AnyRef]]] + if (contents.nonEmpty) { + val content = contents.head + val contentId = content.getOrElse("contentId", "").asInstanceOf[String] + val status = content.getOrElse("status", 0.asInstanceOf[AnyRef]).asInstanceOf[Number].intValue() + val checksum = DeDupHelper.getMessageId(courseId, batchId, userId, contentId, status) + deDupEngine.isUniqueEvent(checksum) + } else false + } else true + } + + def getProgramEvent(eventData: Map[String, AnyRef])( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + cache: DataCache + ): mutable.Iterable[Map[String, AnyRef]] = { + logger.info("EventInfo" + eventData) + var eventInfoMap: mutable.ListBuffer[Map[String, AnyRef]] = mutable.ListBuffer.empty[Map[String, AnyRef]] + val userId: String = eventData.getOrElse(config.userId, "").asInstanceOf[String] + val courseId: String = eventData.getOrElse(config.courseId, "").asInstanceOf[String] + val batchId: String = eventData.getOrElse(config.batchId, "").asInstanceOf[String] + val contentObj: java.util.Map[String, AnyRef] = getCourseInfo(courseId)(metrics, config, cache, httpUtil) + val primaryCategory: String = contentObj.get(config.primaryCategory).asInstanceOf[String] + val parentCollections: List[String] = contentObj.get(config.parentCollections).asInstanceOf[List[String]] + logger.info("Inside Process Method" + primaryCategory + " ParentCollections: " + parentCollections) + if (config.validProgramPrimaryCategory.contains(primaryCategory)) { + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala.map(_.asScala.toMap).toList + val mergedMap = eventData.updated(config.contents, contentConsumption) + eventInfoMap += mergedMap + logger.info("Inside Valid Primary " + mergedMap) + } else if (("Course".equalsIgnoreCase(primaryCategory) || ("Standalone Assessment".equalsIgnoreCase(primaryCategory))) + && !parentCollections.isEmpty) { + for (parentId <- parentCollections) { + val row = getEnrolment(userId, parentId)(metrics) + logger.info("Enrollment: " + row) + if (row != null) { + val contentConsumption = eventData.getOrElse(config.contents, new util.ArrayList[java.util.Map[String, AnyRef]]()).asInstanceOf[util.List[java.util.Map[String, AnyRef]]].asScala + logger.info("contentConsumption: " + contentConsumption) + val filteredContents = contentConsumption.filter(x => x.get("status") == 2).map(_.asScala.toMap).toList + logger.info("filteredContents: " + filteredContents) + if(filteredContents.nonEmpty) { + val eventInfoProgram = Map[String, AnyRef]("contents" -> filteredContents, + "userId" -> userId, + "action" -> "batch-enrolment-update", + "iteration" -> 1.asInstanceOf[Integer], + "batchId" -> row.getString("batchid"), + "courseId" -> parentId) + eventInfoMap += eventInfoProgram + logger.info("EventMapInfoProgram:" + eventInfoProgram) + } + } + } + } else { + logger.error("Not Valid Primary Category: " + primaryCategory + " parentCollections: " + parentCollections) + } + eventInfoMap + } + + def getEnrolment(userId: String, courseId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getCourseInfo(courseId: String)( + metrics: Metrics, + config: ProgramActivityAggregateUpdaterConfig, + cache: DataCache, + httpUtil: HttpUtil + ): java.util.Map[String, AnyRef] = { + val courseMetadata = cache.getWithRetry(courseId) + if (null == courseMetadata || courseMetadata.isEmpty) { + val url = + config.contentReadURL + courseId + "?fields=identifier,name,primaryCategory,parentCollections" + val response = getAPICall(url, "content")(config, httpUtil, metrics) + val courseName = StringContext + .processEscapes( + response.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + response.getOrElse(config.primaryCategory, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = response + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } else { + val courseName = StringContext + .processEscapes( + courseMetadata.getOrElse(config.name, "").asInstanceOf[String] + ) + .filter(_ >= ' ') + val primaryCategory = StringContext + .processEscapes( + courseMetadata + .getOrElse(config.primaryCategory, "") + .asInstanceOf[String] + ) + .filter(_ >= ' ') + val parentCollections = courseMetadata + .getOrElse(config.parentCollections, List.empty[String]).asInstanceOf[List[String]] + val courseInfoMap: java.util.Map[String, AnyRef] = + new java.util.HashMap[String, AnyRef]() + courseInfoMap.put("courseId", courseId) + courseInfoMap.put("courseName", courseName) + courseInfoMap.put("primaryCategory", primaryCategory) + courseInfoMap.put("parentCollections", parentCollections) + courseInfoMap + } + + } + + def getAPICall(url: String, responseParam: String)( + config: ProgramActivityAggregateUpdaterConfig, + httpUtil: HttpUtil, + metrics: Metrics + ): Map[String, AnyRef] = { + val response = httpUtil.get(url, config.defaultHeaders) + if (200 == response.status) { + ScalaJsonUtil + .deserialize[Map[String, AnyRef]](response.body) + .getOrElse("result", Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + .getOrElse(responseParam, Map[String, AnyRef]()) + .asInstanceOf[Map[String, AnyRef]] + } else if ( + 400 == response.status && response.body.contains( + config.userAccBlockedErrCode + ) + ) { + metrics.incCounter(config.skippedEventCount) + logger.error( + s"Error while fetching user details for ${url}: " + response.status + " :: " + response.body + ) + Map[String, AnyRef]() + } else { + throw new Exception( + s"Error from get API : ${url}, with response: ${response}" + ) + } + } + +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala new file mode 100644 index 000000000..cd8cbd385 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressCompleteFunction.scala @@ -0,0 +1,141 @@ +package org.sunbird.job.programaggregate.functions + +import java.util.UUID + +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import com.google.gson.Gson +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.programaggregate.common.DeDupHelper +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.{BaseProcessFunction, Metrics} +import org.sunbird.job.programaggregate.domain.{ActorObject, CollectionProgress, EventContext, EventData, EventObject, TelemetryEvent} +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.CassandraUtil + +import scala.collection.JavaConverters._ + +class ProgramProgressCompleteFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]], val stringTypeInfo: TypeInformation[String], @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessFunction[List[CollectionProgress], String](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramProgressCompleteFunction]) + lazy private val gson = new Gson() + var deDupEngine: DeDupEngine = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + cassandraUtil.close() + deDupEngine.close() + super.close() + } + + override def processElement(events: List[CollectionProgress], context: ProcessFunction[List[CollectionProgress], String]#Context, metrics: Metrics): Unit = { + logger.info("events => "+events) + + val pendingEnrolments = if (config.filterCompletedEnrolments) events.filter {p => + val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) + (row != null && row.getInt("status") != 2) + } else events + logger.info("pendingEnrolments =>"+pendingEnrolments) + + val enrolmentQueries = pendingEnrolments.map(enrolmentComplete => getEnrolmentCompleteQuery(enrolmentComplete)) + logger.info("enrolmentQueries => "+enrolmentQueries) + updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + pendingEnrolments.foreach(e => { + createIssueCertEvent(e, context)(metrics) + generateAuditEvent(e, context)(metrics) + }) + logger.info("posting events completed") + // Create and update the checksum to DeDup store for the input events. + if (config.dedupEnabled) { + events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) + .flatten.foreach(checksum => deDupEngine.storeChecksum(checksum)) + } + } + + override def metricsList(): List[String] = { + List(config.dbReadCount, config.dbUpdateCount, config.enrolmentCompleteCount, config.certIssueEventsCount) + } + + def generateAuditEvent(data: CollectionProgress, context: ProcessFunction[List[CollectionProgress], String]#Context)(implicit metrics: Metrics) = { + val auditEvent = TelemetryEvent( + actor = ActorObject(id = data.userId), + edata = EventData(props = Array("status", "completedon"), `type` = "enrol-complete"), // action values are "start", "complete". + context = EventContext(cdata = Array(Map("type" -> config.courseBatch, "id" -> data.batchId).asJava, Map("type" -> "Course", "id" -> data.courseId).asJava)), + `object` = EventObject(id = data.userId, `type` = "User", rollup = Map[String, String]("l1" -> data.courseId).asJava) + ) + logger.info("audit event =>"+gson.toJson(auditEvent)) + context.output(config.auditEventOutputTag, gson.toJson(auditEvent)) + + } + + def getEnrolment(userId: String, courseId: String, batchId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + .and(QueryBuilder.eq("batchid", batchId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getEnrolmentCompleteQuery(enrolment: CollectionProgress): Update.Where = { + logger.info("Enrolment completed for userId: " + enrolment.userId + " batchId: " + enrolment.batchId) + QueryBuilder.update(config.dbKeyspace, config.dbUserEnrolmentsTable) + .`with`(QueryBuilder.set("status", 2)) + .and(QueryBuilder.set("completedon", enrolment.completedOn)) + .and(QueryBuilder.set("progress", enrolment.progress)) + .and(QueryBuilder.set("contentstatus", enrolment.contentStatus.asJava)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + .where(QueryBuilder.eq("userid", enrolment.userId)) + .and(QueryBuilder.eq("courseid", enrolment.courseId)) + .and(QueryBuilder.eq("batchid", enrolment.batchId)) + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + logger.info("is cassandra cluster available =>"+(null !=cassandraUtil.session)) + val result = cassandraUtil.upsert(cqlBatch.toString) + logger.info("result after update => "+result) + if (result) { + metrics.incCounter(config.dbUpdateCount) + metrics.incCounter(config.enrolmentCompleteCount) + } else { + val msg = "Database update has failed" + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } + + /** + * Generation of Certificate Issue event for the enrolment completed users to validate and generate certificate. + * @param enrolment + * @param context + * @param metrics + */ + def createIssueCertEvent(enrolment: CollectionProgress, context: ProcessFunction[List[CollectionProgress], String]#Context)(implicit metrics: Metrics): Unit = { + val ets = System.currentTimeMillis + val mid = s"""LP.${ets}.${UUID.randomUUID}""" + val event = s"""{"eid": "BE_JOB_REQUEST","ets": ${ets},"mid": "${mid}","actor": {"id": "Course Certificate Generator","type": "System"},"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},"object": {"id": "${enrolment.batchId}_${enrolment.courseId}","type": "CourseCertificateGeneration"},"edata": {"userIds": ["${enrolment.userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${enrolment.batchId}","reIssue": false,"courseId": "${enrolment.courseId}"}}""" + logger.info("o/p event: "+event) + context.output(config.certIssueOutputTag, event) + metrics.incCounter(config.certIssueEventsCount) + } +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala new file mode 100644 index 000000000..677f2973e --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/functions/ProgramProgressUpdateFunction.scala @@ -0,0 +1,99 @@ +package org.sunbird.job.programaggregate.functions + +import com.datastax.driver.core.querybuilder.{QueryBuilder, Select, Update} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.configuration.Configuration +import org.apache.flink.streaming.api.functions.ProcessFunction +import org.slf4j.LoggerFactory +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.programaggregate.common.DeDupHelper +import org.sunbird.job.dedup.DeDupEngine +import org.sunbird.job.programaggregate.domain._ +import org.sunbird.job.programaggregate.task.ProgramActivityAggregateUpdaterConfig +import org.sunbird.job.util.CassandraUtil +import org.sunbird.job.{BaseProcessFunction, Metrics} + +import scala.collection.JavaConverters._ + +class ProgramProgressUpdateFunction(config: ProgramActivityAggregateUpdaterConfig)(implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]], val stringTypeInfo: TypeInformation[String], @transient var cassandraUtil: CassandraUtil = null) + extends BaseProcessFunction[List[CollectionProgress], String](config) { + + private[this] val logger = LoggerFactory.getLogger(classOf[ProgramProgressUpdateFunction]) + var deDupEngine: DeDupEngine = _ + + override def open(parameters: Configuration): Unit = { + super.open(parameters) + cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort) + deDupEngine = new DeDupEngine(config, new RedisConnect(config, Option(config.deDupRedisHost), Option(config.deDupRedisPort)), config.deDupStore, config.deDupExpirySec) + deDupEngine.init() + } + + override def close(): Unit = { + cassandraUtil.close() + deDupEngine.close() + super.close() + } + + override def processElement(events: List[CollectionProgress], context: ProcessFunction[List[CollectionProgress], String]#Context, metrics: Metrics): Unit = { + val pendingEnrolments = if (config.filterCompletedEnrolments) events.filter { p => + val row = getEnrolment(p.userId, p.courseId, p.batchId)(metrics) + (row != null && row.getInt("status") != 2) + } else events + logger.info("The event at progress: "+ events) + logger.info("The pending Enrollment"+ pendingEnrolments) + val enrolmentQueries = pendingEnrolments.map(collectionProgress => getEnrolmentUpdateQuery(collectionProgress)) + logger.info("The pending Enrollment"+ enrolmentQueries) + updateDB(config.thresholdBatchWriteSize, enrolmentQueries)(metrics) + // Create and update the checksum to DeDup store for the input events. + if (config.dedupEnabled) { + events.map(cp => cp.inputContents.map(c => DeDupHelper.getMessageId(cp.courseId, cp.batchId, cp.userId, c, 2))) + .flatten.foreach(checksum => deDupEngine.storeChecksum(checksum)) + } + } + + override def metricsList(): List[String] = { + List(config.dbReadCount, config.dbUpdateCount) + } + + def getEnrolment(userId: String, courseId: String, batchId: String)(implicit metrics: Metrics) = { + val selectWhere: Select.Where = QueryBuilder.select().all() + .from(config.dbKeyspace, config.dbUserEnrolmentsTable). + where() + selectWhere.and(QueryBuilder.eq("userid", userId)) + .and(QueryBuilder.eq("courseid", courseId)) + .and(QueryBuilder.eq("batchid", batchId)) + metrics.incCounter(config.dbReadCount) + cassandraUtil.findOne(selectWhere.toString) + } + + def getEnrolmentUpdateQuery(enrolment: CollectionProgress): Update.Where = { + logger.info("Enrolment updated for userId: " + enrolment.userId + " batchId: " + enrolment.batchId) + QueryBuilder.update(config.dbKeyspace, config.dbUserEnrolmentsTable) + .`with`(QueryBuilder.set("status", 1)) + .and(QueryBuilder.set("progress", enrolment.progress)) + .and(QueryBuilder.set("contentstatus", enrolment.contentStatus.asJava)) + .and(QueryBuilder.set("datetime", System.currentTimeMillis)) + .where(QueryBuilder.eq("userid", enrolment.userId)) + .and(QueryBuilder.eq("courseid", enrolment.courseId)) + .and(QueryBuilder.eq("batchid", enrolment.batchId)) + } + + /** + * Method to update the specific table in a batch format. + */ + def updateDB(batchSize: Int, queriesList: List[Update.Where])(implicit metrics: Metrics): Unit = { + val groupedQueries = queriesList.grouped(batchSize).toList + groupedQueries.foreach(queries => { + val cqlBatch = QueryBuilder.batch() + queries.map(query => cqlBatch.add(query)) + val result = cassandraUtil.upsert(cqlBatch.toString) + if (result) { + metrics.incCounter(config.dbUpdateCount) + } else { + val msg = "Database update has failed" + cqlBatch.toString + logger.error(msg) + throw new Exception(msg) + } + }) + } +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala new file mode 100644 index 000000000..1fcf884e3 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterConfig.scala @@ -0,0 +1,150 @@ +package org.sunbird.job.programaggregate.task + +import com.typesafe.config.Config +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.streaming.api.scala.OutputTag +import org.sunbird.job.BaseJobConfig +import org.sunbird.job.programaggregate.domain.CollectionProgress + +import java.util + +class ProgramActivityAggregateUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "program-activity-aggregate-updater") { + + private val serialVersionUID = 2905979434303791379L + + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + implicit val scalaMapTypeInfo: TypeInformation[Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[Map[String, AnyRef]]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]] = TypeExtractor.getForClass(classOf[List[CollectionProgress]]) + + // Kafka Topics Configuration + val kafkaInputTopic: String = config.getString("kafka.input.topic") + val kafkaAuditEventTopic: String = config.getString("kafka.output.audit.topic") + val kafkaFailedEventTopic: String = config.getString("kafka.output.failed.topic") + val kafkaCertIssueTopic: String = config.getString("kafka.output.certissue.topic") + + override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism") + val activityAggregateUpdaterParallelism: Int = config.getInt("task.activity.agg.parallelism") + val deDupProcessParallelism: Int = config.getInt("task.dedup.parallelism") + val enrolmentCompleteParallelism: Int = config.getInt("task.enrolment.complete.parallelism") + + // Metric List + val totalEventCount = "total-events-count" + val failedEventCount = "failed-events-count" + val dbUpdateCount = "db-update-count" + val dbReadCount = "db-read-count" + val cacheHitCount = "cache-hit-count" + val cacheMissCount = "cache-miss-count" + val batchEnrolmentUpdateEventCount = "batch-enrolment-update-count" + val skipEventsCount = "skipped-events-count" + val processedEnrolmentCount = "processed-enrolment-count" + val enrolmentCompleteCount = "enrolment-complete-count" + val certIssueEventsCount = "cert-issue-events-count" + val retiredCCEventsCount = "retired-consumption-events-count" + + // Cassandra Configurations + val dbUserContentConsumptionTable: String = config.getString("lms-cassandra.consumption.table") + val dbUserActivityAggTable: String = config.getString("lms-cassandra.user_activity_agg.table") + val dbUserEnrolmentsTable: String = config.getString("lms-cassandra.user_enrolments.table") + val dbKeyspace: String = config.getString("lms-cassandra.keyspace") + val dbHost: String = config.getString("lms-cassandra.host") + val dbPort: Int = config.getInt("lms-cassandra.port") + + // Redis Configurations + val nodeStore: Int = config.getInt("redis.database.relationCache.id") // Both LeafNodes And Ancestor nodes + val deDupRedisHost: String = config.getString("dedup-redis.host") + val deDupRedisPort: Int = config.getInt("dedup-redis.port") + val deDupStore: Int = config.getInt("dedup-redis.database.index") + val deDupExpirySec: Int = config.getInt("dedup-redis.database.expiry") + + // Tags + val uniqueConsumptionOutputTagName = "program-unique-consumption-events" + val uniqueConsumptionOutput: OutputTag[Map[String, AnyRef]] = OutputTag[Map[String, AnyRef]](uniqueConsumptionOutputTagName) + val auditEventOutputTagName = "audit-events" + val auditEventOutputTag: OutputTag[String] = OutputTag[String](auditEventOutputTagName) + val failedEventOutputTagName = "failed-events" + val failedEventOutputTag: OutputTag[String] = OutputTag[String](failedEventOutputTagName) + val collectionCompleteOutputTagName = "program-collection-progress-complete-events" + val collectionCompleteOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionCompleteOutputTagName) + val collectionUpdateOutputTagName = "program-collection-progress-update-events" + val collectionUpdateOutputTag: OutputTag[List[CollectionProgress]] = OutputTag[List[CollectionProgress]](collectionUpdateOutputTagName) + val certIssueOutputTagName = "program-certificate-issue-events" + val certIssueOutputTag: OutputTag[String] = OutputTag[String](certIssueOutputTagName) + + // constants + val activityType = "activity_type" + val activityId = "activity_id" + val contextId = "context_id" + val activityUser = "user_id" + val aggLastUpdated = "agg_last_updated" + val agg = "agg" + val courseId = "courseId" + val batchId = "batchId" + val contentId = "contentId" + val progress = "progress" + val contents = "contents" + val contentStatus = "contentStatus" + val userId = "userId" + val status = "status" + val unitActivityType = "course-unit" + val courseActivityType = "course" + val leafNodes = "leafnodes" + val ancestors = "ancestors" + val viewcount = "viewcount" + val completedcount = "completedcount" + val complete = "complete" + val eData = "edata" + val action = "action" + val batchEnrolmentUpdateCode = "batch-enrolment-update" + val routerFn = "RouterFn" + val consumptionDeDupFn= "program-consumption-dedup-process" + val programactivityAggregateUpdaterFn = "program-activity-aggregate-updater-fn" + val partition = "partition" + val courseBatch = "CourseBatch" + val collectionProgressUpdateFn = "progress-update-process" + val collectionCompleteFn = "collection-completion-process" + val aggregates = "aggregates" + + // Consumers + val programActivityAggregateUpdaterConsumer = "program-activity-aggregate-updater-consumer" + + // Producers + val programactivityAggregateUpdaterProducer = "program-activity-aggregate-updater-audit-events-sink" + val enrolmentCompleteEventProducer = "enrolment-complete-audit-sink" + val programactivityAggFailedEventProducer = "program-activity-aggregate-updater-failed-sink" + val certIssueEventProducer = "certificate-issue-event-producer" + + //Thresholds + val thresholdBatchReadInterval: Int = config.getInt("threshold.batch.read.interval") + val thresholdBatchReadSize: Int = config.getInt("threshold.batch.read.size") + val thresholdBatchWriteSize: Int = config.getInt("threshold.batch.write.size") + val windowShards: Int = config.getInt("task.window.shards") + + + // Job specific configurations + val moduleAggEnabled: Boolean = config.getBoolean("activity.module.aggs.enabled") + val dedupEnabled: Boolean = config.getBoolean("activity.input.dedup.enabled") + val statusCacheExpirySec: Int = config.getInt("activity.collection.status.cache.expiry") + val filterCompletedEnrolments: Boolean = if (config.hasPath("activity.filter.processed.enrolments")) config.getBoolean("activity.filter.processed.enrolments") else true + + // Other services configuration + val searchServiceBasePath: String = config.getString("service.search.basePath") + val searchAPIURL = searchServiceBasePath + "/v3/search" + + val contentServiceBase: String = config.getString("service.content.basePath") + val contentReadURL = contentServiceBase + "/content/v3/read/" + + val identifier: String = "identifier" + val primaryCategory: String = "primaryCategory" + val versionKey: String = "versionKey" + val course: String = "Course" + val parentCollections: String = "parentCollections" + val skippedEventCount = "skipped-event-count" + val defaultHeaders = Map[String, String]("Content-Type" -> "application/json") + val userAccBlockedErrCode = "UOS_USRRED0006" + val name: String = "name" + val validProgramPrimaryCategory = List[String]("Program","Curated Program","Blended Program") + + +} diff --git a/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala new file mode 100644 index 000000000..eac0a2783 --- /dev/null +++ b/program-activity-aggregate-updater/src/main/scala/org/sunbird/job/programaggregate/task/ProgramActivityAggregateUpdaterStreamTask.scala @@ -0,0 +1,83 @@ +package org.sunbird.job.programaggregate.task + +import com.typesafe.config.ConfigFactory +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.functions.KeySelector +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.api.java.utils.ParameterTool +import org.apache.flink.streaming.api.scala._ +import org.sunbird.job.programaggregate.domain.CollectionProgress +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.programaggregate.functions.{ProgramActivityAggregatesEnrolUpdateFunction, ProgramActivityAggregatesFunction, ProgramContentConsumptionDeDupFunction, ProgramProgressCompleteFunction, ProgramProgressUpdateFunction} +import org.sunbird.job.util.{FlinkUtil, HttpUtil} + +import java.io.File +import java.util + + +class ProgramActivityAggregateUpdaterStreamTask(config: ProgramActivityAggregateUpdaterConfig, kafkaConnector: FlinkKafkaConnector, httpUtil: HttpUtil) { + def process(): Unit = { + implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config) + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + implicit val stringTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) + implicit val enrolmentCompleteTypeInfo: TypeInformation[List[CollectionProgress]] = TypeExtractor.getForClass(classOf[List[CollectionProgress]]) + + val progressStream = + env.addSource(kafkaConnector.kafkaMapSource(config.kafkaInputTopic)).name(config.programActivityAggregateUpdaterConsumer) + .uid(config.programActivityAggregateUpdaterConsumer).setParallelism(config.kafkaConsumerParallelism) + .rebalance + .process(new ProgramContentConsumptionDeDupFunction(config, httpUtil)).name(config.consumptionDeDupFn) + .uid(config.consumptionDeDupFn).setParallelism(config.deDupProcessParallelism) + .getSideOutput(config.uniqueConsumptionOutput) + .keyBy(new ProgramActivityAggregatorKeySelector(config)) + .countWindow(config.thresholdBatchReadSize) + .process(new ProgramActivityAggregatesEnrolUpdateFunction(config, httpUtil)) + .name(config.programactivityAggregateUpdaterFn) + .uid(config.programactivityAggregateUpdaterFn) + .setParallelism(config.activityAggregateUpdaterParallelism) + + progressStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) + .name(config.programactivityAggregateUpdaterProducer).uid(config.programactivityAggregateUpdaterProducer) + progressStream.getSideOutput(config.failedEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaFailedEventTopic)) + .name(config.programactivityAggFailedEventProducer).uid(config.programactivityAggFailedEventProducer) + + progressStream.getSideOutput(config.collectionUpdateOutputTag).process(new ProgramProgressUpdateFunction(config)) + .name(config.collectionProgressUpdateFn).uid(config.collectionProgressUpdateFn).setParallelism(config.enrolmentCompleteParallelism) + val enrolmentCompleteStream = progressStream.getSideOutput(config.collectionCompleteOutputTag).process(new ProgramProgressCompleteFunction(config)) + .name(config.collectionCompleteFn).uid(config.collectionCompleteFn).setParallelism(config.enrolmentCompleteParallelism) + + enrolmentCompleteStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic)) + .name(config.certIssueEventProducer).uid(config.certIssueEventProducer) + /*enrolmentCompleteStream.getSideOutput(config.auditEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaAuditEventTopic)) + .name(config.enrolmentCompleteEventProducer).uid(config.enrolmentCompleteEventProducer)*/ + + env.execute(config.jobName) + } + +} + +// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster +object ProgramActivityAggregateUpdaterStreamTask { + + def main(args: Array[String]): Unit = { + val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path")) + val config = configFilePath.map { + path => ConfigFactory.parseFile(new File(path)).resolve() + }.getOrElse(ConfigFactory.load("program-activity-aggregate-updater.conf").withFallback(ConfigFactory.systemEnvironment())) + val courseAggregator = new ProgramActivityAggregateUpdaterConfig(config) + val kafkaUtil = new FlinkKafkaConnector(courseAggregator) + val httpUtil = new HttpUtil + val task = new ProgramActivityAggregateUpdaterStreamTask(courseAggregator, kafkaUtil, httpUtil) + task.process() + } + +} +// $COVERAGE-ON$ + +class ProgramActivityAggregatorKeySelector(config: ProgramActivityAggregateUpdaterConfig) extends KeySelector[Map[String, AnyRef], Int] { + private val serialVersionUID = 7267989625042068736L + private val shards = config.windowShards + override def getKey(in: Map[String, AnyRef]): Int = { + in.getOrElse(config.userId, "").asInstanceOf[String].hashCode % shards + } +} diff --git a/program-activity-aggregate-updater/src/test/resources/logback-test.xml b/program-activity-aggregate-updater/src/test/resources/logback-test.xml new file mode 100644 index 000000000..2e5cb5e09 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/logback-test.xml @@ -0,0 +1,18 @@ + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + + diff --git a/program-activity-aggregate-updater/src/test/resources/test.conf b/program-activity-aggregate-updater/src/test/resources/test.conf new file mode 100644 index 000000000..f036ad5e2 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/test.conf @@ -0,0 +1,53 @@ +include "base-test.conf" + +kafka { + input.topic = "sunbirddev.coursebatch.job.request" + output.audit.topic = "sunbirddev.telemetry.raw" + output.failed.topic = "sunbirddev.activity.agg.failed" + output.certissue.topic = "sunbirddev.issue.certificate.request" + groupId = "sunbirddev-activity-aggregate-updater-group" +} + +task { + window.shards = 1 + consumer.parallelism = 1 + dedup.parallelism = 1 + activity.agg.parallelism = 1 + enrolment.complete.parallelism = 1 +} + +lms-cassandra { + keyspace = "sunbird_courses" + consumption.table = "user_content_consumption" + user_activity_agg.table = "user_activity_agg" + user_enrolments.table = "user_enrolments" +} + +redis { + database { + relationCache.id = 10 + } +} + +threshold.batch.read.interval = 60 // In sec +threshold.batch.read.size = 1 +threshold.batch.write.size = 5 + +dedup-redis { + host = localhost + port = 6340 + database.index = 13 + database.expiry = 600 +} + +activity { + module.aggs.enabled = true + input.dedup.enabled = true + collection.status.cache.expiry = 3600 +} + +service { + search { + basePath = "http://search-service:9000" + } +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/test/resources/test.cql b/program-activity-aggregate-updater/src/test/resources/test.cql new file mode 100644 index 000000000..5fff30d90 --- /dev/null +++ b/program-activity-aggregate-updater/src/test/resources/test.cql @@ -0,0 +1,79 @@ +CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1}; + +CREATE TABLE sunbird_courses.user_content_consumption ( + userid text, + courseid text, + batchid text, + contentid text, + completedcount int, + datetime timestamp, + lastaccesstime text, + lastcompletedtime text, + lastupdatedtime text, + progress int, + status int, + viewcount int, + PRIMARY KEY (userid, courseid, batchid, contentid) +) WITH CLUSTERING ORDER BY (courseid ASC, batchid ASC, contentid ASC); + +// EVENT_1 Testcase data +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012299','0126083288437637121','do_1127212344324751361295',100, 2, 3,1) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012300','0126083288437637121','do_1127212344324751361295',100, 2, 2,2) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('8454cb21-3ce9-4e30-85b5-fade097880d8','do_11260735471149056012301','0126083288437637121','do_1127212344324751361295',0, 1,0,1) ; + +//Event_2 Testcase Data +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R1','Batch1','course001',100, 2,1,0) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R2','Batch1','course001',100, 2,1,0) ; +INSERT INTO sunbird_courses.user_content_consumption(userid, contentid, batchid,courseid,progress,status,viewcount,completedcount) VALUES ('user001','do_R3','Batch1','course001',100, 2,1,0) ; + + + +CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg ( + activity_id text, + user_id text, + activity_type text, + context_id text, + agg Map, + aggregates Map, + agg_last_updated Map, + PRIMARY KEY ((activity_type, activity_id), context_id, user_id) +); + +CREATE TABLE sunbird_courses.user_enrolments ( + userid text, + courseid text, + batchid text, + active boolean, + addedby text, + certificates list>>, + completedon timestamp, + completionpercentage int, + contentstatus map, + datetime timestamp, + enrolleddate text, + issued_certificates list>>, + lastreadcontentid text, + lastreadcontentstatus int, + progress int, + status int, + PRIMARY KEY (userid, courseid, batchid) +) WITH CLUSTERING ORDER BY (courseid ASC, batchid ASC) + AND bloom_filter_fp_chance = 0.01 + AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'} + AND comment = '' + AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'} + AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'} + AND crc_check_chance = 1.0 + AND dclocal_read_repair_chance = 0.1 + AND default_time_to_live = 0 + AND gc_grace_seconds = 864000 + AND max_index_interval = 2048 + AND memtable_flush_period_in_ms = 0 + AND min_index_interval = 128 + AND read_repair_chance = 0.0 + AND speculative_retry = '99PERCENTILE'; +CREATE INDEX inx_ues_status ON sunbird_courses.user_enrolments (status); +CREATE INDEX inx_ues_certs ON sunbird_courses.user_enrolments (values(certificates)); + + +INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('user001', 'course001', 'Batch1', 1) diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala new file mode 100644 index 000000000..a26f3426b --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/fixture/EventFixture.scala @@ -0,0 +1,194 @@ +package org.sunbird.job.fixture + +object EventFixture { + + /** + * case-1. Inserting a first course which is having 3 leaf nodes in the redis database and cassandra database + * does not contains this course id. + * courseId = + * ============BE_JOB_REQUEST_CONTENTS========== + * do_1127212344324751361295 - course + * do_course_unit1 - unit1 + * do_11260735471149056012299 - resource + * do_course_unit2 - unit2 + * do_11260735471149056012300 - resource + * do_course_unit3 - unit3 + * do_11260735471149056012301 - resource + * do_11260735471149056012300 -resource + * + * + * ============== content status in the event ====== + * do_11260735471149056012299 - 2 + * do_11260735471149056012301 - 1 + * do_11260735471149056012300 - 1 + * ============== content status in the database(content-consumption) + * do_11260735471149056012299 - 2 + * do_11260735471149056012301 - 1 + * do_11260735471149056012300 - 2 + * + * ============== Computation ============== + * unit level computation + * do_11260735471149056012299:ansestor -> do_course_unit1, do_1127212344324751361295 + * do_course_unit1:do_1127212344324751361295:leafnodes: do_11260735471149056012299 + * lefNodesSize = 1, completed = 1 + * 1/1 = 100% + * + * do_11260735471149056012301:ansestor -> do_course_unit3,do_1127212344324751361295 + * do_course_unit3:do_1127212344324751361295:leafNodes -> do_11260735471149056012301,do_11260735471149056012300 + * leafNodesSize = 2,completed 1 + * 1/2 = 50% + * + * do_11260735471149056012300:ansestor -> do_course_unit3,do_course_unit2,do_1127212344324751361295 + * do_course_unit3:do_1127212344324751361295:leafNodes -> do_11260735471149056012301,do_11260735471149056012300 + * leafNodesSize = 2, completed = 1 + * 1/2 = 50% + * + * do_course_unit2:do_1127212344324751361295:leafNodes -> do_11260735471149056012300 + * leafNodesSize = 1, completed = 1 + * 1/1 = 100% + * + * course level progress computation + * do_1127212344324751361295:leafNodes = do_11260735471149056012299, do_11260735471149056012301, do_11260735471149056012300 + */ + + val EVENT_1: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-enrolment-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val courseLeafNodes = Map("do_1127212344324751361295:do_1127212344324751361295:leafnodes" -> List("do_11260735471149056012299", "do_11260735471149056012300", "do_11260735471149056012301")) + val unitLeafNodes_1 = Map("do_1127212344324751361295:do_course_unit1:leafnodes" -> List("do_11260735471149056012299")) + val unitLeafNodes_2 = Map("do_1127212344324751361295:do_course_unit2:leafnodes" -> List("do_11260735471149056012300")) + val unitLeafNodes_3 = Map("do_1127212344324751361295:do_course_unit3:leafnodes" -> List("do_11260735471149056012301", "do_11260735471149056012300")) + + val ancestorsResource_1 = Map("do_1127212344324751361295:do_11260735471149056012299:ancestors" -> List("do_course_unit1", "do_1127212344324751361295")) + val ancestorsResource_2 = Map("do_1127212344324751361295:do_11260735471149056012300:ancestors" -> List("do_course_unit2", "do_course_unit3", "do_1127212344324751361295")) + val ancestorsResource_3 = Map("do_1127212344324751361295:do_11260735471149056012301:ancestors" -> List("do_course_unit3", "do_1127212344324751361295")) + + val CASE_1:Map[String, AnyRef] = Map("event" -> EVENT_1, "cacheData" -> List(courseLeafNodes, + unitLeafNodes_1, unitLeafNodes_2,unitLeafNodes_3, ancestorsResource_1,ancestorsResource_2,ancestorsResource_3)) + + + + val EVENT_2: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R1","status":2},{"contentId":"do_R2","status":1},{"contentId":"do_R3","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + + /** **** course structure **** + * + * case2: When all resource progress is 2 in the content-consumption table + * + * course001 - course + * unit1 - unit1 + * do_R1 - Resource + * do_R3 - Resource + * unit2 - Unit2 + * do_R2 - Resource + * do_R3 - Resource + * + * ============== content status in the event ====== + * do_R1 - 1 + * do_R3 - 1 + * do_R2 - 1 + * ============== content status in the database(content-consumption) + * do_R1 - 2 + * do_R2 - 2 + * do_R3 - 2 + * + * //Unit Level + * course001:do_R1:ansestor => unit1,course001 + * unit1:leafNodes -> do_R1,do_R3 + * output:leafNodesSize = 2, completed=2 + * course001:do_R2:ansestor => unit2,course001 + * unit2:leafNodes -> do_R2,do_R3 + * output:leafNodesSize = 2, completed = 2 + * course001:do_R3:ansestor => unit1, unit2,course001 + * unit1:leafNodes -> do_R1,do_R3 + * unit2:leafNodes -> do_R2,do_R3 + * output:leafNodes=2, completed=2 + * // CourseLevel + * output:LeafNodes =3, Completed =3 + * + * + */ + val e2_courseLeafNodes = Map("course001:course001:leafnodes" -> List("do_R1", "do_R3", "do_R2")) + val e2_unitLeafNodes_1 = Map("course001:unit1:leafnodes" -> List("do_R1", "do_R3")) + val e2_unitLeafNodes_2 = Map("course001:unit2:leafnodes" -> List("do_R2", "do_R3")) + + val e2_ancestorsResource_1 = Map("course001:do_R1:ancestors" -> List("unit1", "course001")) + val e2_ancestorsResource_2 = Map("course001:do_R3:ancestors" -> List("unit1", "unit2", "course001")) + val e2_ancestorsResource_3 = Map("course001:do_R2:ancestors" -> List("unit2", "course001")) + + val CASE_2:Map[String, AnyRef] = Map("event" -> EVENT_2, "cacheData" -> List(e2_courseLeafNodes, + e2_unitLeafNodes_1, e2_unitLeafNodes_2,e2_ancestorsResource_1, e2_ancestorsResource_2,e2_ancestorsResource_3)) + + /** * + * + * Case3: When resource data is not available in the content_consumption table and user_activity_agg + * + * C11 - course + * unit11 - unit1 + * R11 - Resource + * R22 - Resource + * unit22 - Unit2 + * R11 - Resource + * + * ============== content status in the event ====== + * R11 - 2 + * R22 - 2 + * ============== content status in the database(content-consumption) + * Data is not available + * + * //Unit Level + * C11:R11:ansestor => unit11,unit22,C11 + * unit11:leafNodes -> R11,R22 + * output:leafNodesSize = 2, completed=2 + * unit11:leafNodes -> R11 + * output:leafNodesSize = 1, completed=1 + * C11:R22:ansestor => unit11,C11 + * unit11:leafNodes -> R11,R22 + * output:leafNodesSize = 2, completed = 2 + * // CourseLevel + * output:LeafNodes =2, Completed =2 + * + */ + + val e3_courseLeafNodes = Map("C11:C11:leafnodes" -> List("R11", "R22")) + val e3_unitLeafNodes_1 = Map("C11:unit11:leafnodes" -> List("R11", "R22")) + val e3_unitLeafNodes_2 = Map("C11:unit22:leafnodes" -> List("R11")) + + val e3_ancestorsResource_1 = Map("C11:R11:ancestors" -> List("unit11", "C11")) + val e3_ancestorsResource_2 = Map("C11:R11:ancestors" -> List("unit22", "C11")) + val e3_ancestorsResource_3 = Map("C11:R22:ancestors" -> List("unit11", "C11")) + + val EVENT_3: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"R11","status":2},{"contentId":"R22","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"B11","userId":"U11","courseId":"C11"}} + |""".stripMargin + + val CASE_3:Map[String, AnyRef] = Map("event" -> EVENT_3, "cacheData" -> List(e3_courseLeafNodes, e3_unitLeafNodes_1, e3_unitLeafNodes_2,e3_ancestorsResource_1, e3_ancestorsResource_2, e3_ancestorsResource_3) ) + + val EVENT_4: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-enrolment-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val EVENT_5: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac8","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_11260735471149056012299","status":2},{"contentId":"do_11260735471149056012300","status":1},{"contentId":"do_11260735471149056012301","status":1}],"action":"batch-update","iteration":1,"batchId":"0126083288437637121","userId":"8454cb21-3ce9-4e30-85b5-fade097880d8","courseId":"do_1127212344324751361295"}} + |""".stripMargin + + val CC_EVENT1: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac81","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R1","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + val CC_EVENT2: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac82","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R2","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin + val CC_EVENT3: String = + """ + |{"eid":"BE_JOB_REQUEST","ets":1563788371969,"mid":"LMS.1563788371969.590c5fa0-0ce8-46ed-bf6c-681c0a1fdac83","actor":{"type":"System","id":"Course Batch Updater"},"context":{"pdata":{"ver":"1.0","id":"org.sunbird.platform"}},"object":{"type":"CourseBatchEnrolment","id":"0126083288437637121_8454cb21-3ce9-4e30-85b5-fade097880d8"},"edata":{"contents":[{"contentId":"do_R3","status":2}],"action":"batch-enrolment-update","iteration":1,"batchId":"Batch1","userId":"user001","courseId":"course001"}} + |""".stripMargin +} \ No newline at end of file diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala new file mode 100644 index 000000000..cd881c3ab --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/BaseActivityAggregateTestSpec.scala @@ -0,0 +1,59 @@ +package org.sunbird.job.spec + +import java.util + +import org.apache.flink.streaming.api.functions.sink.SinkFunction + + +class AuditEventSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + AuditEventSink.values.add(value) + } + } +} + +object AuditEventSink { + val values: util.List[String] = new util.ArrayList() +} + +class FailedEventSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + FailedEventSink.values.add(value) + } + } +} + +object FailedEventSink { + val values: util.List[String] = new util.ArrayList() +} + +class SuccessEvent extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + SuccessEventSink.values.add(value) + } + } +} + +object SuccessEventSink { + val values: util.List[String] = new util.ArrayList() +} + + +class CertificateIssuedEventsSink extends SinkFunction[String] { + + override def invoke(value: String): Unit = { + synchronized { + CertificateIssuedEvents.values.add(value) + } + } +} + +object CertificateIssuedEvents { + val values: util.List[String] = new util.ArrayList() +} diff --git a/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala new file mode 100644 index 000000000..2194ab23d --- /dev/null +++ b/program-activity-aggregate-updater/src/test/scala/org/sunbird/job/spec/ProgramActivityAggregateUpdaterTaskTestSpec.scala @@ -0,0 +1,219 @@ +package org.sunbird.job.spec + +import java.util +import com.datastax.driver.core.Row +import com.google.gson.Gson +import com.typesafe.config.{Config, ConfigFactory} +import org.apache.flink.api.common.typeinfo.TypeInformation +import org.apache.flink.api.java.typeutils.TypeExtractor +import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration +import org.apache.flink.streaming.api.functions.source.SourceFunction +import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext +import org.apache.flink.test.util.MiniClusterWithClientResource +import org.cassandraunit.CQLDataLoader +import org.cassandraunit.dataset.cql.FileCQLDataSet +import org.cassandraunit.utils.EmbeddedCassandraServerHelper +import org.mockito.Mockito +import org.mockito.Mockito._ +import org.sunbird.job.cache.RedisConnect +import org.sunbird.job.connector.FlinkKafkaConnector +import org.sunbird.job.fixture.EventFixture +import org.sunbird.job.programaggregate.task.{ProgramActivityAggregateUpdaterConfig, ProgramActivityAggregateUpdaterStreamTask} +import org.sunbird.job.util.{CassandraUtil, HTTPResponse, HttpUtil} +import org.sunbird.spec.{BaseMetricsReporter, BaseTestSpec} +import redis.clients.jedis.Jedis +import redis.embedded.RedisServer + +import scala.collection.mutable +import scala.collection.JavaConverters._ + +class ProgramActivityAggregateUpdaterTaskTestSpec extends BaseTestSpec { + + implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]]) + + val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder() + .setConfiguration(testConfiguration()) + .setNumberSlotsPerTaskManager(1) + .setNumberTaskManagers(1) + .build) + + var redisServer: RedisServer = _ + redisServer = new RedisServer(6340) + redisServer.start() + var jedis: Jedis = _ + val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable()) + val gson = new Gson() + val config: Config = ConfigFactory.load("test.conf") + val courseAggregatorConfig: ProgramActivityAggregateUpdaterConfig = new ProgramActivityAggregateUpdaterConfig(config) + val mockHttpUtil: HttpUtil = mock[HttpUtil](Mockito.withSettings().serializable()) + + var cassandraUtil: CassandraUtil = _ + + val requestBody = s"""{ + | "request": { + | "filters": { + | "objectType": "Collection", + | "identifier": "course001", + | "status": ["Live", "Unlisted", "Retired"] + | }, + | "fields": ["status"] + | } + |}""".stripMargin + + override protected def beforeAll(): Unit = { + super.beforeAll() + val redisConnect = new RedisConnect(courseAggregatorConfig) + jedis = redisConnect.getConnection(courseAggregatorConfig.nodeStore) + EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L) + cassandraUtil = new CassandraUtil(courseAggregatorConfig.dbHost, courseAggregatorConfig.dbPort) + val session = cassandraUtil.session + + val dataLoader = new CQLDataLoader(session) + dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true)) + // Clear the metrics + testCassandraUtil(cassandraUtil) + BaseMetricsReporter.gaugeMetrics.clear() + jedis.flushDB() + flinkCluster.before() + updateRedis(jedis, EventFixture.CASE_1.asInstanceOf[Map[String, AnyRef]]) + updateRedis(jedis, EventFixture.CASE_2.asInstanceOf[Map[String, AnyRef]]) + updateRedis(jedis, EventFixture.CASE_3.asInstanceOf[Map[String, AnyRef]]) + } + + override protected def afterAll(): Unit = { + super.afterAll() + try { + EmbeddedCassandraServerHelper.cleanEmbeddedCassandra() + redisServer.stop() + } catch { + case ex: Exception => { + } + } + flinkCluster.after() + } + + def initialize() { + when(mockKafkaUtil.kafkaMapSource(courseAggregatorConfig.kafkaInputTopic)).thenReturn(new CompleteContentConsumptionMapSource) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaAuditEventTopic)).thenReturn(new AuditEventSink) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaFailedEventTopic)).thenReturn(new FailedEventSink) + when(mockKafkaUtil.kafkaStringSink(courseAggregatorConfig.kafkaCertIssueTopic)).thenReturn(new CertificateIssuedEventsSink) + } + + "Activity Aggregator " should " compute and update enrolment as completed when all the content consumption data processed" in { + initialize() + new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, new HttpUtil).process() + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.totalEventCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.batchEnrolmentUpdateEventCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.dbReadCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.dbUpdateCount}").getValue() should be(6) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.cacheHitCount}").getValue() should be(18) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.processedEnrolmentCount}").getValue() should be(3) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.enrolmentCompleteCount}").getValue() should be(1) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.failedEventCount}").getValue() should be(0) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.skipEventsCount}").getValue() should be(0) + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.cacheMissCount}").getValue() should be(0) + + AuditEventSink.values.size() should be(4) + AuditEventSink.values.forEach(event => { + println("AUDIT_TELEMETRY_EVENT: " + event) + }) + jedis.select(courseAggregatorConfig.deDupStore) + val deDupKeys = jedis.keys("*") + println("DeDup Keys:" + deDupKeys) + deDupKeys.size() should be (3) + jedis.select(courseAggregatorConfig.nodeStore) + } + + "Activity Aggregator " should " throw exception when the cache not available for root collection" in { + jedis.select(courseAggregatorConfig.nodeStore) + jedis.flushAll() + when(mockHttpUtil.post(courseAggregatorConfig.searchAPIURL, requestBody)).thenReturn(HTTPResponse(200, """{"id":"api.v1.search","ver":"1.0","ts":"2020-12-16T12:37:40.283Z","params":{"resmsgid":"7c4cf0b0-3f9b-11eb-9b0c-abcfbdf41bc3","msgid":"7c4b1bf0-3f9b-11eb-9b0c-abcfbdf41bc3","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"count":1,"content":[{"identifier":"course001","objectType":"Content","status":"Live"}]}}""")) + initialize() + + val activityAggTask = new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, mockHttpUtil) + the [Exception] thrownBy { + activityAggTask.process() + } should have message "Job execution failed." + + // De-dup should not save the keys for which the processing failed. + // This will help in processing the same data after restart. + jedis.select(courseAggregatorConfig.deDupStore) + jedis.keys("*").size() should be (0) + jedis.select(courseAggregatorConfig.nodeStore) + + FailedEventSink.values.forEach(event => { + println("FAILED_EVENT_DATA: " + event) + }) + // failedEventSink.values.size() should be (2) + } + + ignore should " skip the retired collection consumption events" in { + jedis.select(courseAggregatorConfig.nodeStore) + jedis.flushAll() + reset(mockHttpUtil) + when(mockHttpUtil.post(courseAggregatorConfig.searchAPIURL, requestBody)).thenReturn(HTTPResponse(200, """{"id":"api.v1.search","ver":"1.0","ts":"2020-12-16T12:37:40.283Z","params":{"resmsgid":"7c4cf0b0-3f9b-11eb-9b0c-abcfbdf41bc3","msgid":"7c4b1bf0-3f9b-11eb-9b0c-abcfbdf41bc3","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"count":1,"content":[{"identifier":"course001","objectType":"Content","status":"Retired"}]}}""")) + initialize() + + val activityAggTask = new ProgramActivityAggregateUpdaterStreamTask(courseAggregatorConfig, mockKafkaUtil, mockHttpUtil) + activityAggTask.process() + + jedis.select(courseAggregatorConfig.deDupStore) + jedis.keys("*").size() should be (0) + jedis.select(courseAggregatorConfig.nodeStore) + + BaseMetricsReporter.gaugeMetrics(s"${courseAggregatorConfig.jobName}.${courseAggregatorConfig.retiredCCEventsCount}").getValue() should be(3) + + } + + def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = { + cassandraUtil.reconnect() + } + + def updateRedis(jedis: Jedis, testData: Map[String, AnyRef]) { + testData.get("cacheData").map(data => { + data.asInstanceOf[List[Map[String, AnyRef]]].map(cacheData => { + cacheData.map(x => { + x._2.asInstanceOf[List[String]].foreach(d => { + jedis.sadd(x._1, d) + }) + }) + }) + }) + } + + def readFromCassandra(event: String): util.List[Row] = { + val event1_primaryCols = getPrimaryCols(gson.fromJson(event, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala.asJava) + val query = s"select * from sunbird_courses.user_activity_agg where context_id='cb:${event1_primaryCols.get("batchid").get}' and user_id='${event1_primaryCols.get("userid").get}' ALLOW FILTERING;" + cassandraUtil.find(query) + } + + def readFromContentConsumptionTable(event: String): util.List[Row] = { + val event1_primaryCols = getPrimaryCols(gson.fromJson(event, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala.asJava) + val query = s"select * from sunbird_courses.user_content_consumption where userid='${event1_primaryCols.get("userid").get}' and batchid='${event1_primaryCols.get("batchid").get}' and courseid='${event1_primaryCols.get("courseid").get}' ALLOW FILTERING;" + cassandraUtil.find(query) + } + + + def getPrimaryCols(event: util.Map[String, AnyRef]): mutable.Map[String, String] = { + val eventData = event.get("edata").asInstanceOf[util.Map[String, AnyRef]] + val primaryFields = List("userid", "courseid", "batchid") + eventData.asScala.map(v => (v._1.toLowerCase, v._2)).filter(x => primaryFields.contains(x._1)).asInstanceOf[mutable.Map[String, String]] + } +} + +private class CompleteContentConsumptionMapSource extends SourceFunction[util.Map[String, AnyRef]] { + + override def run(ctx: SourceContext[util.Map[String, AnyRef]]) { + ctx.collect(jsonToMap(EventFixture.CC_EVENT1)) + ctx.collect(jsonToMap(EventFixture.CC_EVENT2)) + ctx.collect(jsonToMap(EventFixture.CC_EVENT3)) + } + + override def cancel() = {} + + def jsonToMap(json: String): util.Map[String, AnyRef] = { + val gson = new Gson() + gson.fromJson(json, new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]] + } + +}