diff --git a/frontend/src/app/workspace/component/menu/menu.component.html b/frontend/src/app/workspace/component/menu/menu.component.html index a21e4d56429..48a65b473ef 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.html +++ b/frontend/src/app/workspace/component/menu/menu.component.html @@ -149,6 +149,16 @@ nz-icon nzType="info-circle"> + @@ -456,3 +466,115 @@ + + +
+
+ Notebook to Workflow +
+ + +

+ This tool converts a Python Jupyter Notebook into a Texera workflow using LLM capabilities. After you submit a + notebook, the LLM service will generate a corresponding Texera workflow. The conversion time depends on the + notebook’s complexity and can take 1–5 minutes. Once the process is complete, the workflow workspace will reload + with: +

+
    +
  1. + The generated workflow ready to use (Note: you will still need to upload the dataset and connect it to the + workflow). +
    2. +
  2. A floating Jupyter window containing the uploaded notebook for reference.
    3. +
+

+ Feel free to navigate away from this tab while you wait for the workflow to generate. Please do not close the + window. +

+ + + + + Upload Python Jupyter Notebook + + +
+ + + + + + Selected file: {{ importForm.get('file')?.value?.name }} + +
+ + + + + + Select Model Type + + + + + + + + + + + + + + + + + + API Key + + + + + +
+ diff --git a/frontend/src/app/workspace/component/menu/menu.component.ts b/frontend/src/app/workspace/component/menu/menu.component.ts index 6375fbcee4e..3cb60766204 100644 --- a/frontend/src/app/workspace/component/menu/menu.component.ts +++ b/frontend/src/app/workspace/component/menu/menu.component.ts @@ -17,9 +17,21 @@ * under the License. */ -import { DatePipe, Location, NgIf, NgFor, NgTemplateOutlet } from "@angular/common"; -import { Component, ElementRef, Input, OnDestroy, OnInit, ViewChild } from "@angular/core"; +import { DatePipe, Location, NgIf, NgFor, NgTemplateOutlet, AsyncPipe, NgOptimizedImage } from "@angular/common"; +import { + Component, + ElementRef, + Input, + OnDestroy, + OnInit, + ViewChild, + Output, + EventEmitter, + TemplateRef, + ViewContainerRef, +} from "@angular/core"; import { Router, RouterLink } from "@angular/router"; +import { FormBuilder, FormGroup, Validators } from "@angular/forms"; import { UserService } from "../../../common/service/user/user.service"; import { DEFAULT_WORKFLOW_NAME, @@ -33,7 +45,7 @@ import { WorkflowActionService } from "../../service/workflow-graph/model/workfl import { ExecutionState } from "../../types/execute-workflow.interface"; import { WorkflowWebsocketService } from "../../service/workflow-websocket/workflow-websocket.service"; import { WorkflowResultExportService } from "../../service/workflow-result-export/workflow-result-export.service"; -import { catchError, debounceTime, filter, mergeMap, tap } from "rxjs/operators"; +import { catchError, debounceTime, filter, mergeMap, switchMap, tap } from "rxjs/operators"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { WorkflowUtilService } from "../../service/workflow-graph/util/workflow-util.service"; import { WorkflowVersionService } from "../../../dashboard/service/user/workflow-version/workflow-version.service"; @@ -43,7 +55,7 @@ import { saveAs } from "file-saver"; import { NotificationService } from "src/app/common/service/notification/notification.service"; import { OperatorMenuService } from "../../service/operator-menu/operator-menu.service"; import { CoeditorPresenceService } from "../../service/workflow-graph/model/coeditor-presence.service"; -import { firstValueFrom, of, Subscription, timer } from "rxjs"; +import { firstValueFrom, map, of, Subscription, timer } from "rxjs"; import { isDefined } from "../../../common/util/predicate"; import { NzModalService } from "ng-zorro-antd/modal"; import { ResultExportationComponent } from "../result-exportation/result-exportation.component"; @@ -74,6 +86,13 @@ import { NzPopoverDirective } from "ng-zorro-antd/popover"; import { NzSwitchComponent } from "ng-zorro-antd/switch"; import { NzBadgeComponent } from "ng-zorro-antd/badge"; import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; +import { JupyterPanelService } from "../../service/jupyter-panel/jupyter-panel.service"; +import { v4 as uuidv4 } from "uuid"; +import { Notebook } from "../../service/notebook-migration/migration-llm"; +import { NotebookMigrationService } from "../../service/notebook-migration/notebook-migration.service"; +import { NzFormModule } from "ng-zorro-antd/form"; +import { NzSelectModule } from "ng-zorro-antd/select"; +import { ReactiveFormsModule } from "@angular/forms"; /** * MenuComponent is the top level menu bar that shows @@ -122,6 +141,11 @@ import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; NzTooltipDirective, DatePipe, NzSpaceCompactComponent, + NzFormModule, + AsyncPipe, + NzSelectModule, + ReactiveFormsModule, + NgOptimizedImage, ], }) export class MenuComponent implements OnInit, OnDestroy { @@ -147,6 +171,8 @@ export class MenuComponent implements OnInit, OnDestroy { @Input() public currentExecutionName: string = ""; // reset executionName @Input() public particularVersionDate: string = ""; // placeholder for the metadata information of a particular workflow version @ViewChild("workflowNameInput") workflowNameInput: ElementRef | undefined; + // Emit an event to parent component (workspace) when AI generation starts or stops + @Output() public setWaitingForLLM = new EventEmitter(); // variable bound with HTML to decide if the running spinner should show public runButtonText = "Run"; @@ -167,6 +193,9 @@ export class MenuComponent implements OnInit, OnDestroy { @ViewChild(ComputingUnitSelectionComponent) computingUnitSelectionComponent!: ComputingUnitSelectionComponent; + public importForm: FormGroup; + @ViewChild("importNotebookModal", { static: true }) importModalTpl!: TemplateRef; + constructor( public executeWorkflowService: ExecuteWorkflowService, public workflowActionService: WorkflowActionService, @@ -189,7 +218,12 @@ export class MenuComponent implements OnInit, OnDestroy { private panelService: PanelService, private computingUnitStatusService: ComputingUnitStatusService, protected config: GuiConfigService, - private router: Router + private router: Router, + private fb: FormBuilder, + private modal: NzModalService, + private viewContainerRef: ViewContainerRef, + private jupyterPanelService: JupyterPanelService, + private notebookMigrationService: NotebookMigrationService ) { workflowWebsocketService .subscribeToEvent("ExecutionDurationUpdateEvent") @@ -219,6 +253,13 @@ export class MenuComponent implements OnInit, OnDestroy { // Subscribe to computing unit this.subscribeToComputingUnitSelection(); this.subscribeToComputingUnitStatus(); + + this.importForm = this.fb.group({ + description: [""], + file: [null, Validators.required], + model: [""], + apiKey: [""], + }); } public ngOnInit(): void { @@ -594,6 +635,172 @@ export class MenuComponent implements OnInit, OnDestroy { this.workflowActionService.deleteOperatorsAndLinks(allOperatorIDs); } + public get pythonNotebookMigrationEnabled(): boolean { + return this.config.env.pythonNotebookMigrationEnabled; + } + + openImportNotebookModal(): void { + const models$ = this.notebookMigrationService.getAvailableModels().pipe( + tap({ + error: () => this.notificationService.error("Failed to fetch models"), + }) + ); + + const modalRef = this.modal.create({ + nzTitle: "AI Generate Workflow from Python Notebook", + nzContent: this.importModalTpl, + nzViewContainerRef: this.viewContainerRef, + nzWidth: 700, + nzData: { + models$: models$, + }, + nzFooter: [ + { + label: "Cancel", + onClick: () => { + modalRef.close(); + }, + }, + { + label: "Submit", + type: "primary", + disabled: () => !this.importForm.valid, + onClick: () => { + const file: NzUploadFile = this.importForm.get("file")?.value; + const model: string = this.importForm.get("model")?.value; + const apiKey: string = this.importForm.get("apiKey")?.value; + this.onClickImportNotebook(file, model, apiKey); + modalRef.close(); // close after submit too + }, + }, + ], + }); + } + + public beforeUpload = (file: NzUploadFile) => { + this.importForm.patchValue({ file }); + this.importForm.get("file")?.markAsDirty(); + this.importForm.get("file")?.updateValueAndValidity(); + return false; // prevent auto upload + }; + + public onClickImportNotebook = (file: NzUploadFile, model: string, apiKey: string): boolean => { + const reader = new FileReader(); + + // Check if the file is a Jupyter notebook based on its extension + const fileExtension = file.name.split(".").pop()?.toLowerCase(); + if (fileExtension !== "ipynb") { + this.notificationService.error("Please upload a valid Jupyter Notebook (.ipynb) file."); + return false; + } + + this.setWaitingForLLM.emit(true); // start loading + + // Read the notebook file as text + reader.readAsText(file as any); + reader.onload = async () => { + try { + const result = reader.result; + if (typeof result !== "string") { + throw new Error("File content is not a valid string."); + } + + // Parse the content of the .ipynb file (it's in JSON format) + const notebookContent = JSON.parse(result) as Notebook; + + // Validate the notebook structure + if (!notebookContent || !Array.isArray(notebookContent.cells)) { + throw new Error("Invalid notebook structure."); + } + + // Add UUID's to each cell in the notebook + for (const cell of notebookContent.cells) { + if (!cell.metadata) { + cell.metadata = {}; + } + cell.metadata.uuid = uuidv4(); + } + + // Send Notebook JSON to pod to open in jupyterlab + await this.notebookMigrationService.sendNotebookToJupyter(notebookContent); + + // Get workflow and mapping from LLM + await this.notebookMigrationService + .sendToAIGenerateWorkflow(notebookContent, model, apiKey) + .then(result => { + if (result) { + const { workflowContent, mappingContent } = result; + + const fileExtensionIndex = file.name.lastIndexOf("."); + var workflowName: string; + if (fileExtensionIndex === -1) { + workflowName = file.name; + } else { + workflowName = file.name.substring(0, fileExtensionIndex); + } + if (workflowName.trim() === "") { + workflowName = DEFAULT_WORKFLOW_NAME; + } + + const workflow: Workflow = { + content: workflowContent, + name: `${workflowName}_GENERATED_BY_LLM`, + isPublished: 0, + description: undefined, + wid: undefined, + creationTime: undefined, + lastModifiedTime: undefined, + readonly: false, + }; + + this.workflowPersistService + .persistWorkflow(workflow) + .pipe( + switchMap((updatedWorkflow: Workflow) => { + const mappingID = "mapping_wid_" + updatedWorkflow.wid; + + this.notebookMigrationService.setMapping(mappingID, mappingContent); + + return this.notebookMigrationService + .storeNotebookAndMapping(updatedWorkflow.wid, 1, mappingContent, notebookContent) + .pipe(map(() => updatedWorkflow)); + }), + untilDestroyed(this) + ) + .subscribe({ + next: updatedWorkflow => { + this.workflowActionService.reloadWorkflow(updatedWorkflow, true); + this.jupyterPanelService.openPanel("JupyterNotebookPanel"); + this.notificationService.success("Successfully generated workflow and mapping from notebook."); + }, + error: (err: unknown) => { + this.notificationService.error("Failed to import notebook, check console for detailed error"); + console.error("Import notebook failed:", err); + }, + complete: () => { + this.setWaitingForLLM.emit(false); + }, + }); + } else { + console.error("Result is undefined"); + } + }) + .catch(error => { + this.notificationService.error("Error while communicating with LLM, check console for details"); + console.error("Error while fetching data from LLM: ", error); + }) + .finally(() => { + this.setWaitingForLLM.emit(false); // stop loading + }); + } catch (error) { + this.notificationService.error("Failed to import the notebook."); + console.error(error); + } + }; + + return false; // Prevent automatic upload handling + }; + public onClickImportWorkflow = (file: NzUploadFile): boolean => { const reader = new FileReader(); reader.readAsText(file as any); diff --git a/frontend/src/app/workspace/component/workspace.component.html b/frontend/src/app/workspace/component/workspace.component.html index c54446fb318..1ad2592e96b 100644 --- a/frontend/src/app/workspace/component/workspace.component.html +++ b/frontend/src/app/workspace/component/workspace.component.html @@ -21,7 +21,22 @@ + nzTip="Loading workflow..."> + + +
@@ -29,8 +44,10 @@ + [pid]="pid" + (setWaitingForLLM)="onWaitingForLLMChanged($event)"> + { + this.updateElapsedTime(); + }, 1000); + } + + stopTimer() { + clearInterval(this.timerInterval); + this.timerInterval = null; + this.startTime = null; + } + + updateElapsedTime() { + this.cdRef.detectChanges(); + } + + get formattedElapsedTime(): string { + if (!this.startTime) return "00:00"; + const diff = Date.now() - this.startTime; + const minutes = Math.floor(diff / 60000); + const seconds = Math.floor((diff % 60000) / 1000); + return `${minutes}:${seconds.toString().padStart(2, "0")}`; + } } diff --git a/frontend/src/assets/notebook_migration_tool/tool_popup_diagram.png b/frontend/src/assets/notebook_migration_tool/tool_popup_diagram.png new file mode 100644 index 00000000000..438cebc88c9 Binary files /dev/null and b/frontend/src/assets/notebook_migration_tool/tool_popup_diagram.png differ