feat(python-notebook-migration): add notebook-migration-service microservice in backend

build.sbt

@@ -167,6 +167,16 @@ lazy val WorkflowExecutionService = (project in file("amber"))
   )
   .configs(Test)
   .dependsOn(DAO % "test->test", Auth % "test->test") // test scope dependency
+lazy val NotebookMigrationService = (project in file("notebook-migration-service"))
+  .dependsOn(Auth, Config, DAO)
+  .settings(asfLicensingSettings)
+  .settings(
+    dependencyOverrides ++= Seq(
+      // override it as io.dropwizard 4 require 2.16.1 or higher
+      "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.17.0"
+    )
+  )
+  .dependsOn(DAO % "test->test") // test scope dependency
 
 // root project definition
 lazy val TexeraProject = (project in file("."))
@@ -184,7 +194,8 @@ lazy val TexeraProject = (project in file("."))
     ConfigService,
     FileService,
     WorkflowCompilingService,
-    WorkflowExecutionService
+    WorkflowExecutionService,
+    NotebookMigrationService
   )
   .settings(
     name := "texera",

common/config/src/main/resources/storage.conf

@@ -146,4 +146,11 @@ storage {
         password = "postgres"
         password = ${?STORAGE_JDBC_PASSWORD}
     }
+
+    # Configurations of the JupyterLab service
+    # Default values are provided for each field, which you don't need to change if you deployed Jupyter via docker-compose.yml in notebook-migration-service/src/main/resources/docker-compose.yml
+    jupyter {
+        url = "http://localhost:9100"
+        url = ${?STORAGE_JUPYTER_URL}
+    }
 }

common/config/src/main/scala/org/apache/texera/amber/config/StorageConfig.scala

@@ -132,4 +132,7 @@ object StorageConfig {
   val ENV_S3_REGION = "STORAGE_S3_REGION"
   val ENV_S3_AUTH_USERNAME = "STORAGE_S3_AUTH_USERNAME"
   val ENV_S3_AUTH_PASSWORD = "STORAGE_S3_AUTH_PASSWORD"
+
+  // Jupyter
+  val jupyterURL: String = conf.getString("storage.jupyter.url")
 }

frontend/proxy.config.json

@@ -10,6 +10,11 @@
     "secure": false,
     "changeOrigin": true
   },
+  "/api/notebook-migration": {
+    "target": "http://localhost:9098",
+    "secure": false,
+    "changeOrigin": true
+  },
 "/api/models": {
     "target": "http://localhost:9096",
     "secure": false,

notebook-migration-service/build.sbt

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import scala.collection.Seq
+
+name := "notebook-migration-service"
+
+
+enablePlugins(JavaAppPackaging)
+
+// Enable semanticdb for Scalafix
+ThisBuild / semanticdbEnabled := true
+ThisBuild / semanticdbVersion := scalafixSemanticdb.revision
+
+// Manage dependency conflicts by always using the latest revision
+ThisBuild / conflictManager := ConflictManager.latestRevision
+
+// Restrict parallel execution of tests to avoid conflicts
+Global / concurrentRestrictions += Tags.limit(Tags.Test, 1)
+
+/////////////////////////////////////////////////////////////////////////////
+// Compiler Options
+/////////////////////////////////////////////////////////////////////////////
+
+// Scala compiler options
+Compile / scalacOptions ++= Seq(
+  "-Xelide-below", "WARNING",       // Turn on optimizations with "WARNING" as the threshold
+  "-feature",                       // Check feature warnings
+  "-deprecation",                   // Check deprecation warnings
+  "-Ywarn-unused:imports"           // Check for unused imports
+)
+
+/////////////////////////////////////////////////////////////////////////////
+// Version Variables
+/////////////////////////////////////////////////////////////////////////////
+
+val dropwizardVersion = "4.0.7"
+val mockitoVersion = "5.4.0"
+val assertjVersion = "3.24.2"
+
+/////////////////////////////////////////////////////////////////////////////
+// Test-related Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+libraryDependencies ++= Seq(
+  "org.scalamock" %% "scalamock" % "5.2.0" % Test,                   // ScalaMock
+  "org.scalatest" %% "scalatest" % "3.2.17" % Test,                  // ScalaTest
+  "io.dropwizard" % "dropwizard-testing" % dropwizardVersion % Test, // Dropwizard Testing
+  "org.mockito" % "mockito-core" % mockitoVersion % Test,            // Mockito for mocking
+  "org.assertj" % "assertj-core" % assertjVersion % Test,            // AssertJ for assertions
+  "com.novocode" % "junit-interface" % "0.11" % Test                // SBT interface for JUnit
+)
+
+/////////////////////////////////////////////////////////////////////////////
+// Dependencies
+/////////////////////////////////////////////////////////////////////////////
+
+// Core Dependencies
+libraryDependencies ++= Seq(
+  "io.dropwizard" % "dropwizard-core" % dropwizardVersion,
+  "io.dropwizard" % "dropwizard-auth" % dropwizardVersion, // Dropwizard Authentication module
+  "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.18.6"
+)

notebook-migration-service/project/build.properties

@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+sbt.version = 1.12.9

notebook-migration-service/src/main/resources/logback.xml

@@ -0,0 +1,55 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+<configuration>
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <!-- encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder
+            by default -->
+        <encoder>
+            <pattern>[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n
+            </pattern>
+        </encoder>
+    </appender>
+
+
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/notebook-migration-service.log</file>
+        <immediateFlush>true</immediateFlush>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>logs/notebook-migration-service-%d{yyyy-MM-dd}.log.gz</fileNamePattern>
+        </rollingPolicy>
+        <encoder>
+            <pattern>[%date{ISO8601}] [%level] [%logger] [%thread] - %msg %n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="ASYNC" class="ch.qos.logback.classic.AsyncAppender">
+        <queueSize>8192</queueSize>
+        <neverBlock>true</neverBlock>
+        <appender-ref ref="FILE"/>
+    </appender>
+
+    <root level="${TEXERA_SERVICE_LOG_LEVEL:-INFO}">
+        <appender-ref ref="ASYNC"/>
+        <appender-ref ref="STDOUT"/>
+    </root>
+    <logger name="org.apache" level="WARN"/>
+    <logger name="httpclient" level="WARN"/>
+    <logger name="io.grpc.netty" level="WARN"/>
+</configuration>

notebook-migration-service/src/main/resources/notebook-migration-service-web-config.yaml

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+server:
+  applicationConnectors:
+    - type: http
+      port: 9098
+  adminConnectors: []
+  requestLog:
+    type: classic
+    appenders: []
+
+logging:
+  level: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+  appenders:
+    - type: console
+      threshold: ${TEXERA_SERVICE_LOG_LEVEL:-INFO}
+    - type: file
+      currentLogFilename: logs/notebook-migration-service.log
+      archive: true
+      archivedLogFilenamePattern: logs/notebook-migration-service-%d.log.gz
+      archivedFileCount: 5

notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationService.scala

@@ -0,0 +1,80 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.texera.service
+
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import com.typesafe.scalalogging.LazyLogging
+import io.dropwizard.configuration.{EnvironmentVariableSubstitutor, SubstitutingSourceProvider}
+import io.dropwizard.core.Application
+import io.dropwizard.core.setup.{Bootstrap, Environment}
+import org.apache.texera.amber.config.StorageConfig
+import org.apache.texera.auth.RequestLoggingFilter
+import org.apache.texera.dao.SqlServer
+import java.nio.file.Path
+import org.apache.texera.service.resource.NotebookMigrationResource
+
+class NotebookMigrationService
+    extends Application[NotebookMigrationServiceConfiguration]
+    with LazyLogging {
+  override def initialize(bootstrap: Bootstrap[NotebookMigrationServiceConfiguration]): Unit = {
+    // enable environment variable substitution in YAML config
+    bootstrap.setConfigurationSourceProvider(
+      new SubstitutingSourceProvider(
+        bootstrap.getConfigurationSourceProvider,
+        new EnvironmentVariableSubstitutor(false)
+      )
+    )
+    // Register Scala module to Dropwizard default object mapper
+    bootstrap.getObjectMapper.registerModule(DefaultScalaModule)
+
+    SqlServer.initConnection(
+      StorageConfig.jdbcUrl,
+      StorageConfig.jdbcUsername,
+      StorageConfig.jdbcPassword
+    )
+  }
+
+  override def run(
+      configuration: NotebookMigrationServiceConfiguration,
+      environment: Environment
+  ): Unit = {
+    // Serve backend at /api
+    environment.jersey.setUrlPattern("/api/*")
+
+    environment.jersey.register(classOf[NotebookMigrationResource])
+
+    // Route request logs through SLF4J, controlled by TEXERA_SERVICE_LOG_LEVEL
+    RequestLoggingFilter.register(environment.getApplicationContext)
+  }
+}
+object NotebookMigrationService {
+  def main(args: Array[String]): Unit = {
+    val notebookMigrationPath = Path
+      .of(sys.env.getOrElse("TEXERA_HOME", "."))
+      .resolve("notebook-migration-service")
+      .resolve("src")
+      .resolve("main")
+      .resolve("resources")
+      .resolve("notebook-migration-service-web-config.yaml")
+      .toAbsolutePath
+      .toString
+
+    // Start the Dropwizard application
+    new NotebookMigrationService().run("server", notebookMigrationPath)
+  }
+}

notebook-migration-service/src/main/scala/org/apache/texera/service/NotebookMigrationServiceConfiguration.scala

@@ -0,0 +1,22 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.texera.service
+
+import io.dropwizard.core.Configuration
+
+class NotebookMigrationServiceConfiguration extends Configuration {}