Production Deploy

polyphemus/lib/client/jsx/workflow/metis-form-components.tsx

@@ -152,7 +152,7 @@ export const TableFormat = ({value,update}:ScriptItem) => <Select displayEmpty v
 export const BlankTable = ({value, update, modelName, classes}:ScriptItem) => {
   const {models} = useContext(MagmaContext);
   const parentAttribute: string = models ? models[modelName]?.template?.parent : '__error__'
-  const isTable: boolean = parentAttribute!='project' && models[parentAttribute]?.template?.attributes[modelName].attribute_type=='table'
+  const isTable: boolean = modelName!='project' && models[parentAttribute]?.template?.attributes[modelName].attribute_type=='table'
 
   if (value==undefined) {
     update(true)
@@ -229,7 +229,7 @@ export const ColumnMap = ({value, update, modelName, classes}:ScriptItem) => {
   const idAttribute: string = models?.[modelName]?.template?.identifier || '__error__'
   const parentAttribute: string = models?.[modelName]?.template?.parent || '__error__'
   // Determine if table.
-  const isTable: boolean = parentAttribute!='project' && models[parentAttribute]?.template?.attributes[modelName].attribute_type=='table'
+  const isTable: boolean = modelName!='project' && models[parentAttribute]?.template?.attributes[modelName].attribute_type=='table'
   const autoAttribute: string = isTable ? parentAttribute : idAttribute
 
   useEffect( () => {

polyphemus/lib/data_eng/jobs/metis_linker.rb

@@ -21,16 +21,18 @@ def bucket_name
     config['config']['bucket_name']
   end
 
+  def updates_only?
+    !!(runtime_config['config'] || {})[:updates_only]
+  end
+
   def pre(context)
-    context[:start_time] = Time.at(fetch_last_scan).to_datetime.iso8601
+    context[:start_time] = updates_only? ? fetch_last_scan : Time.at(0).to_datetime.iso8601
     context[:end_time] = Time.now.to_datetime.iso8601
     true
   end
 
   # Process method containing the main File Discovery ETL logic
   def process(context)
-    last_scan = fetch_last_scan
-
     rules = gnomon_client.project_rules(project_name).rules
 
     project_def = magma_client.retrieve(project_name: project_name)
@@ -53,7 +55,7 @@ def process(context)
 
     summary = <<EOT
 ===============================
-Upload Summary : #{context[:start_time]} -> #{context[:end_time]} 
+Upload Window: #{context[:start_time]} -> #{context[:end_time]} 
 Models: #{response.models.model_keys.join(', ')}
 Committed to Magma: #{!loader.config.dry_run?}
 Autolinked Parent Identifiers: #{loader.config.autolink?}
@@ -94,7 +96,7 @@ def fetch_last_scan
        workflow_config_id,
        state: [:end_time]
       )
-      return response['end_time'].to_i
+      return response['end_time']
     rescue Etna::Error => e
       return 0
     end

polyphemus/lib/data_eng/workflow_manifests/metis_linker.rb

@@ -11,6 +11,11 @@ def self.as_json
           commit: {
             type: 'boolean',
             description: 'Commit results to Magma'
+          },
+          updates_only: {
+            type: 'boolean',
+            description: 'Only link files updated on Metis since the last loader run',
+            default: nil
           }
         },
         workflow_path: '/app/workflows/argo/metis_linker/workflow.yaml'

polyphemus/lib/etls/metis/loader.rb

@@ -242,6 +242,10 @@ def identifier(model_name, path)
         return match ? match[0] : nil
       end
 
+      def updates_only?
+        !!@params[:updates_only]
+      end
+
       def dry_run?
         ! @params[:commit]
       end

polyphemus/spec/data_eng/metis_linker_job_spec.rb

@@ -110,6 +110,38 @@ def labors_config(scripts, bucket_name="pics", autolink=true)
     stub_request(:post, "#{POLYPHEMUS_HOST}/api/workflows/labors/run/update/#{run_id}").to_return(body: "{}")
   end
 
+  context 'tail' do
+    it 'filters the tail if updates_only' do
+      stub_request(:post, "https://polyphemus.test/api/workflows/labors/run/previous/1").to_return(
+        body: { end_time: Time.now.iso8601 }.to_json,
+        headers: { 'Content-Type': "application/json" },
+      )
+
+      job = MetisLinkerJob.new(TEST_TOKEN, config, runtime_config.merge('config' => { 'updates_only' => true }))
+
+      context = {}
+
+      job.pre(context)
+
+      expect(context[:start_time]).to be > '1970'
+    end
+
+    it 'includes all files if not updates_only' do
+      stub_request(:post, "https://polyphemus.test/api/workflows/labors/run/previous/1").to_return(
+        body: { end_time: Time.now.iso8601 }.to_json,
+        headers: { 'Content-Type': "application/json" },
+      )
+
+      job = MetisLinkerJob.new(TEST_TOKEN, config, runtime_config.merge('config' => { 'commit' => true }))
+
+      context = {}
+
+      job.pre(context)
+
+      expect(context[:start_time]).to eq('1970-01-01T00:00:00+00:00')
+    end
+  end
+
   context 'linking' do
     it 'successfully links records' do
       stub_request(:post, "https://polyphemus.test/api/workflows/labors/run/previous/1").to_return(

vulcan/lib/client/jsx/api_types.ts

@@ -137,6 +137,7 @@ interface WorkspaceMinusInconsistent {
   last_job_status: {[k: string]: StatusStringFine} | null;
   last_run_id: number | null;
   tags: string[] | null;
+  vignette?: string
 }
 export interface WorkspaceRaw extends WorkspaceMinusInconsistent {
   vulcan_config: VulcanConfigElement[];
@@ -145,7 +146,6 @@ export interface WorkspaceRaw extends WorkspaceMinusInconsistent {
 export interface Workspace extends WorkspaceMinusInconsistent {
   vulcan_config: VulcanConfig;
   // project?: string;
-  vignette?: string;
   thumbnails?: string[];
 }
 

vulcan/lib/client/jsx/components/workspace/drawers/vignette.tsx

@@ -9,12 +9,12 @@ export default function Vignette({}) {
   const [text, setText] = useState('');
 
   useEffect(() => {
-    if ('vignette.md' in state.status.file_contents) {
+    if (!!state.workspace && !!state.workspace.vignette) {
       setText(
-        state.status.file_contents['vignette.md']
+        state.workspace.vignette
       );
     }
-  }, [state.status.file_contents]);
+  }, [state.workspace]);
 
   return (
     <div

vulcan/lib/client/jsx/components/workspace/workspace_manager.tsx

@@ -409,8 +409,8 @@ export default function WorkspaceManager() {
             icon='book'
             className='header-btn vignette'
             label='Workflow'
-            title={'vignette.md' in state.status.file_contents ? 'Workflow Readme' : 'Workflow Readme Unavailable'}
-            disabled={!('vignette.md' in state.status.file_contents)}
+            title={!!state.workspace?.vignette ? 'Workflow Readme' : 'Workflow Readme Unavailable'}
+            disabled={!state.workspace?.vignette}
             onClick={() => {setWorkspaceHelpIsOpen(true)}}
           />
           <ReactModal

vulcan/lib/path.rb

@@ -38,7 +38,7 @@ def self.dl_config(workspace_dir)
       "#{workspace_dir}/dl_config.yaml"
     end
 
-    def self.default_snakemake_config(workspace_dir)
+    def self.stub_snakemake_config(workspace_dir)
       "#{workspace_dir}/default-config.json"
     end
 

vulcan/lib/server/controllers/vulcan_v2_controller.rb

@@ -69,8 +69,8 @@ def create_workspace
           Vulcan::Path.dl_config(workspace_dir), 
           Vulcan::Path.dl_config_yaml(@escaped_params[:project_name], task_token, Vulcan.instance.config(:magma)[:host])
         )
-        config = @remote_manager.read_yaml_file(Vulcan::Path.default_snakemake_config(workspace_dir))
-        target_mapping = @snakemake_manager.generate_target_mapping(workspace_dir, config)
+        stub_config = @remote_manager.read_yaml_file(Vulcan::Path.stub_snakemake_config(workspace_dir))
+        target_mapping = @snakemake_manager.generate_target_mapping(workspace_dir, stub_config)
         obj = Vulcan::Workspace.create(
           workflow_id: workflow.id,
           name: @params[:workspace_name],
@@ -83,10 +83,19 @@ def create_workspace
           created_at: Time.now,
           updated_at: Time.now
         )
+        vulcan_config = @remote_manager.read_yaml_file(Vulcan::Path.vulcan_config(workspace_dir))
+        config_defaults = vulcan_config.select{|c| c.key?("default") && c.key?("output") && c["output"].key?("params")}
+        .map{ |c| [c["output"]["params"][0], c["default"]]}
+        .to_h
+        if !config_defaults.empty?
+          workspace_state = Vulcan::WorkspaceState.new(obj, @snakemake_manager, @remote_manager)
+          config_hash = Digest::MD5.hexdigest(config_defaults.to_json + Time.now.to_i.to_s)
+          workspace_state.set_config(config_defaults, config_hash)
+        end
         response = {
           workspace_id: obj.id,
           workflow_id: obj.workflow_id,
-          vulcan_config: @remote_manager.read_yaml_file(Vulcan::Path.vulcan_config(workspace_dir)),
+          vulcan_config: vulcan_config,
           dag: obj.dag,
           dag_flattened: Vulcan::Snakemake::Inference.flatten_adjacency_list(obj.dag),
           file_dag: Vulcan::Snakemake::Inference.file_graph(target_mapping)
@@ -148,6 +157,11 @@ def get_workspace
       last_run_id: last_run ? last_run.id : nil,
       last_job_status: last_run ? slurm_status : nil
     })
+    # Fetch vignette
+    vignette_path = "#{workspace.path}/resources/vignette.md"
+    if @remote_manager.file_exists?(vignette_path)
+      response[:vignette] = @remote_manager.read_file_to_memory(vignette_path)
+    end
     success_json(response)
   end
   # Update workspace name and tags
@@ -192,34 +206,13 @@ def save_config
       # We always create a new config, even if the params / files are the same for a previous config.
       # It is safest to let snakemake figure out what the "current state" is.
 
-      params_hash = Digest::MD5.hexdigest(workflow_params_json + Time.now.to_i.to_s) # generate random hash 
-      config = Vulcan::Config.first(workspace_id: workspace.id, hash: params_hash)
-
-      # We always overwrite the default config with the incoming params
-      default_config = @remote_manager.read_json_file(Vulcan::Path.default_snakemake_config(workspace.path))
-      config_values = JSON.parse(default_config.to_json).merge(JSON.parse(workflow_params_json))
-      config_path = Vulcan::Path.workspace_config_path(workspace.path, params_hash)
-      @remote_manager.write_file(config_path, config_values.to_json)
-
       # Anytime a config is saved, we need to clean up UI targets
       workspace_state = Vulcan::WorkspaceState.new(workspace, @snakemake_manager, @remote_manager)
       workspace_state.remove_existing_ui_targets(@params[:uiFilesSent], @params[:paramsChanged])
 
-      # Generate the future state
-      available_files = workspace_state.get_available_files
-      state = workspace_state.state(workflow_params.keys.map(&:to_s), config_path, available_files)
-
-      config = Vulcan::Config.create(
-          workspace_id: workspace.id,
-          path: config_path,
-          hash: params_hash,
-          input_files: "{#{available_files.join(',')}}",
-          input_params: JSON.parse(workflow_params_json),
-          state: state.to_json,
-          created_at: Time.now,
-          updated_at: Time.now
-        )
-
+      params_hash = Digest::MD5.hexdigest(workflow_params_json + Time.now.to_i.to_s) # generate random hash
+      config_values = JSON.parse(workflow_params_json)
+      config = workspace_state.set_config(config_values, params_hash)
       success_json(
         {
           config_id: config.id,
@@ -247,15 +240,19 @@ def run_workflow
     begin
       raise Etna::TooManyRequests.new("workflow is still running...") if @snakemake_manager.snakemake_is_running?(workspace.path)
       config = Vulcan::Config.where(id: @params[:config_id]).first
+      ### Todo: We don't actually check if the workspace is in a similar state to when the given config was established
       unless config
         msg = "Config for workspace: #{workspace.path} does not exist."
         raise Etna::BadRequest.new(msg)
       end
       # Build snakemake command for execution using stored future_state
       state = config.state
+      # Ensure stub config has proper values, and stubbed other values needed for successfully running.
+      stub_path = Vulcan::Path.stub_snakemake_config(workspace.path)
+      @remote_manager.write_file(stub_path, @remote_manager.read_json_file(stub_path).merge(config.input_params).to_json)
       command = Vulcan::Snakemake::CommandBuilder.new
       command.targets = state['files']['planned']
-      command.options[:config_path] = config.path
+      command.options[:config_path] = stub_path
       command.options[:profile_path] = Vulcan::Path.profile_dir(workspace.path, "default") # only one profile for now
       slurm_run_uuid = @snakemake_manager.run_snakemake(workspace.path, command.build)
       log = @snakemake_manager.get_snakemake_log(workspace.path, slurm_run_uuid)
@@ -408,7 +405,9 @@ def get_state
     begin
       workspace_state = Vulcan::WorkspaceState.new(workspace, @snakemake_manager, @remote_manager)
       available_files = workspace_state.get_available_files
-      state = workspace_state.state(config.input_params.keys.map(&:to_s), config.path, available_files)
+      stub_path = Vulcan::Path.stub_snakemake_config(workspace.path)
+      @remote_manager.write_file(stub_path, @remote_manager.read_json_file(stub_path).merge(config.input_params).to_json)
+      state = workspace_state.state(config.input_params.keys.map(&:to_s), stub_path, available_files)
       success_json(state)
     rescue => e
       Vulcan.instance.logger.log_error(e)
@@ -434,7 +433,7 @@ def update_target_mapping
 
     begin
       # Read the current default config from the workspace
-      config = @remote_manager.read_yaml_file(Vulcan::Path.default_snakemake_config(workspace.path))
+      config = @remote_manager.read_yaml_file(Vulcan::Path.stub_snakemake_config(workspace.path))
 
       # Regenerate the target mapping
       target_mapping = @snakemake_manager.generate_target_mapping(workspace.path, config)

vulcan/lib/snakemake_remote_manager.rb

@@ -161,7 +161,7 @@ def get_dag(dir)
         # Therefore, we adopt a convention in the config.yaml - such that all input files
         # be defined starting with the string "output/". We can then create dummy files and run
         # our meta commands.
-        config_path = Vulcan::Path.default_snakemake_config(Shellwords.escape(dir))
+        config_path = Vulcan::Path.stub_snakemake_config(Shellwords.escape(dir))
         config = @remote_manager.read_yaml_file(config_path)
 
         # Identify input files and create them with "DUMMY FILE" content

vulcan/lib/workspace_state.rb

@@ -13,23 +13,30 @@ def state(params, config_path, available_files)
         params, 
         available_files
       )
-      # Use dry run to get the all the files, jobs that snakemake ACTUALLY plans to generate
-      # This gets us the files and jobs planned and completed, also note that these are just the compute jobs and files, not the UI jobs and files.
-      command = Vulcan::Snakemake::CommandBuilder.new
-      command.targets = all_targets
-      command.options[:config_path] = config_path
-      command.options[:profile_path] = Vulcan::Path.profile_dir(@workspace.path, "default")
-      command.options[:dry_run] = true
-      command.options[:summary] = true
-      # === COMPUTE STATE (from snakemake dry run) ===
-      # Get compute files/jobs in all three states: planned, completed, unscheduled
-      # Note that these are just the compute jobs and files, not the UI jobs and files.
-
-      dry_run_info = @snakemake_manager.dry_run_snakemake_files(@workspace.path, command.build)
-      files_planned = dry_run_info[:files_scheduled].to_set.to_a
-      jobs_planned = dry_run_info[:jobs_scheduled].to_set.to_a
-      files_completed = dry_run_info[:files_completed].to_set.to_a
-      jobs_completed = dry_run_info[:jobs_completed].to_set.to_a
+      if all_targets.empty?
+        files_planned = []
+        jobs_planned = []
+        files_completed = []
+        jobs_completed = []
+      else
+        # Use dry run to get the all the files, jobs that snakemake ACTUALLY plans to generate
+        # This gets us the files and jobs planned and completed, also note that these are just the compute jobs and files, not the UI jobs and files.
+        command = Vulcan::Snakemake::CommandBuilder.new
+        command.targets = all_targets
+        command.options[:config_path] = config_path
+        command.options[:profile_path] = Vulcan::Path.profile_dir(@workspace.path, "default")
+        command.options[:dry_run] = true
+        command.options[:summary] = true
+        # === COMPUTE STATE (from snakemake dry run) ===
+        # Get compute files/jobs in all three states: planned, completed, unscheduled
+        # Note that these are just the compute jobs and files, not the UI jobs and files.
+
+        dry_run_info = @snakemake_manager.dry_run_snakemake_files(@workspace.path, command.build)
+        files_planned = dry_run_info[:files_scheduled].to_set.to_a
+        jobs_planned = dry_run_info[:jobs_scheduled].to_set.to_a
+        files_completed = dry_run_info[:files_completed].to_set.to_a
+        jobs_completed = dry_run_info[:jobs_completed].to_set.to_a
+      end
 
       Vulcan.instance.logger.debug("Files scheduled: #{files_planned}")
       Vulcan.instance.logger.debug("Jobs scheduled: #{jobs_planned}")
@@ -108,7 +115,6 @@ def remove_existing_ui_targets(ui_files_written, params_changed)
       end
     end
 
-
     def get_available_files
       output_files = @remote_manager.list_files(Vulcan::Path.workspace_output_dir(@workspace.path))
         .map { |file| "output/#{file}" }
@@ -117,6 +123,29 @@ def get_available_files
       output_files + resources_files
     end
 
+    def set_config(config_values, config_hash)
+      config_path = Vulcan::Path.workspace_config_path(@workspace.path, config_hash)
+      available_files = get_available_files
+      # Write file to workspace
+      @remote_manager.write_file(config_path, config_values.to_json)
+      # Fill filled values into the stub_config
+      stub_path = Vulcan::Path.stub_snakemake_config(@workspace.path)
+      @remote_manager.write_file(stub_path, @remote_manager.read_json_file(stub_path).merge(config_values).to_json)
+      # Determine state afterward
+      state = state(config_values.keys.map(&:to_s), stub_path, available_files)
+      # Create db entry
+      Vulcan::Config.create(
+        workspace_id: @workspace.id,
+        path: config_path,
+        hash: config_hash,
+        input_files: "{#{available_files.join(',')}}",
+        input_params: config_values,
+        state: state.to_json,
+        created_at: Time.now,
+        updated_at: Time.now
+      )
+    end
+
     private
 
     def categorize_ui_targets