From 2d7f3447d5cef21cff78d458b0c100c7f84efc42 Mon Sep 17 00:00:00 2001 From: Jillian Rowe Date: Thu, 6 Jul 2017 12:08:15 +0400 Subject: [PATCH] fixed stats --- .../run/Rules/Directives/Types/Path.pm | 11 + .../Command/run/Utils/Files/ResolveDeps.pm | 3 +- .../Command/run/Utils/Files/TrackChanges.pm | 224 +++--------------- lib/BioX/Workflow/Command/stats.pm | 11 +- 4 files changed, 48 insertions(+), 201 deletions(-) diff --git a/lib/BioX/Workflow/Command/run/Rules/Directives/Types/Path.pm b/lib/BioX/Workflow/Command/run/Rules/Directives/Types/Path.pm index d63d1bd..3d0e04f 100644 --- a/lib/BioX/Workflow/Command/run/Rules/Directives/Types/Path.pm +++ b/lib/BioX/Workflow/Command/run/Rules/Directives/Types/Path.pm @@ -38,6 +38,17 @@ has 'outdir' => ( documentation => q(Output directories for rules and processes), ); +has 'cwd' => ( + is => 'rw', + isa => Path, + coerce => 1, + required => 0, + default => sub { cwd(); }, + predicate => 'has_cwd', + clearer => 'clear_cwd', + documentation => q(Placeholder for the cwd.), +); + =head3 INPUT OUTPUT Special variables that can have input/output diff --git a/lib/BioX/Workflow/Command/run/Utils/Files/ResolveDeps.pm b/lib/BioX/Workflow/Command/run/Utils/Files/ResolveDeps.pm index 19a33d6..9a50ecf 100644 --- a/lib/BioX/Workflow/Command/run/Utils/Files/ResolveDeps.pm +++ b/lib/BioX/Workflow/Command/run/Utils/Files/ResolveDeps.pm @@ -44,8 +44,7 @@ sub add_graph { my $self = shift; my $cond = shift; - for my $pair ( $self->files_pairs ) { - my $file = $pair->[0]; + for my $file ( $self->all_files ) { if ( !exists $self->rule_deps->{ $self->rule_name }->{$cond}->{$file} ) { $self->rule_deps->{ $self->rule_name }->{$cond}->{$file} = 1; diff --git a/lib/BioX/Workflow/Command/run/Utils/Files/TrackChanges.pm b/lib/BioX/Workflow/Command/run/Utils/Files/TrackChanges.pm index 9e1640b..48dd596 100644 --- a/lib/BioX/Workflow/Command/run/Utils/Files/TrackChanges.pm +++ b/lib/BioX/Workflow/Command/run/Utils/Files/TrackChanges.pm @@ -7,7 +7,6 @@ use File::stat; use Time::localtime; use File::Basename; use DateTime::Format::Strptime; -# use Memoize; with 'BioX::Workflow::Command::Utils::Files::TrackChanges'; @@ -15,52 +14,46 @@ with 'BioX::Workflow::Command::Utils::Files::TrackChanges'; Files just for this rule +##TODO Make this a hash? =cut -has 'files' => ( - traits => ['Hash'], - is => 'rw', - isa => 'HashRef', - default => sub { {} }, - handles => { - files_pairs => 'kv', - clear_files => 'clear', - }, -); +# has 'files' => ( +# traits => ['Hash'], +# is => 'rw', +# isa => 'HashRef', +# default => sub { {} }, +# handles => { +# files_pairs => 'kv', +# clear_files => 'clear', +# }, +# ); - -#TODO Update this -#No longer seen_modify - since we are not checking for timestamps anymore -has 'seen_modify' => ( - traits => ['Hash'], +has 'files' => ( + traits => ['Array'], is => 'rw', - isa => 'HashRef', - default => sub { {} }, + isa => 'ArrayRef', + default => sub { [] }, handles => { - seen_modify_pairs => 'kv', - clear_seen_modify => 'clear', + has_files => 'count', + all_files => 'elements', + push_files => 'push', + sort_files => 'sort_in_place', + uniq_files => 'uniq', }, + clearer => 'clear_files', ); sub walk_FILES { my $self = shift; my $attr = shift; - $self->seen_modify->{local} = {}; - - # my $mod_input = $self->pre_FILES( $attr, 'INPUT' ); $self->pre_FILES( $attr, 'INPUT' ); $self->add_graph('INPUT'); $self->clear_files; - # $self->app_log->info('Between checks...'); - # my $mod_output = $self->pre_FILES( $attr, 'OUTPUT' ); - $self->pre_FILES($attr, 'OUTPUT'); + $self->pre_FILES( $attr, 'OUTPUT' ); $self->add_graph('OUTPUT'); $self->clear_files; - # $self->local_attr->{_modified} = 1 if $mod_input; - - # TODO add check for when we have actually modified the output } sub pre_FILES { @@ -68,188 +61,31 @@ sub pre_FILES { my $attr = shift; my $cond = shift; - # $self->app_log->info( 'Beginning modification checks for ' . $cond ); - # REF check? walk { wanted => sub { $self->walk_INPUT(@_) } }, $attr->$cond; - - # Once we get to the input we want to see if we are processing a new file or no - # my $modify = $self->iterate_FILES; - $self->iterate_FILES; - - # return $modify; + $self->uniq_files; + $self->sort_files; } -sub iterate_FILES { - my $self = shift; +=head3 walk_INPUT - # my $modify = 0; - for my $pair ( $self->files_pairs ) { - my $file = $pair->[0]; - $self->seen_modify->{local}->{$file} = 1; - # if ( $self->seen_modify->{all}->{$file} ) { - # my $tmodify = $self->seen_modify->{all}->{$file}; - # $modify = 1 if $tmodify; - # $self->update_seen( $file, $modify ); - # next; - # } - # elsif ( !-e $file ) { - # $self->update_seen( $file, 1 ); - # $modify = 1; - # next; - # } - # - # if ( $self->process_file($file) ) { - # $modify = 1; - # } - } - - # return $modify; -} - -=head3 process_file +walk the INPUT/OUTPUT and catch all Path::Tiny references =cut -# sub process_file { -# my $self = shift; -# my $file = shift; -# -# $self->seen_modify->{local}->{$file} = 1; -# # my $details = File::Details->new($file); -# # my $mtime = ctime( stat($file)->mtime ); -# # -# # if ( exists $self->track_files->{$file}->{mtime} ) { -# # -# # # Check to see if we have a difference -# # my $p_mtime = $self->track_files->{$file}->{mtime}; -# # if ( compare_mtimes( $p_mtime, $mtime ) ) { -# # # $self->flag_for_process( $file, $p_mtime, $mtime ); -# # $self->update_seen( $file, 1 ); -# # -# # return 1; -# # } -# # else { -# # # This is the only time we should return 0 -# # $self->update_seen( $file, 0 ); -# # return 0; -# # } -# # } -# # else { -# # $self->update_seen( $file, 1 ); -# # return 1; -# # } -# } - -# This is really more of a sanity check -sub flag_for_process { - my $self = shift; - my $file = shift; - my $p_mtime = shift; - my $mtime = shift; - - my $basename = basename($file); - - # #TO LOG OR NOT TO LOG - # $self->app_log->warn( - # 'File ' . $file . ' has been modified since your last analysis.' ); - # $self->app_log->warn( $basename - # . ":\n\tLast Recorded Modification:\t" - # . $p_mtime - # . "\n\tMost Recent Modification:\t" - # . $mtime ); - - # TODO We only want this in run... - # TODO Add an override here - # if ( !$self->check_select && !$self->use_timestamps ) { - # $self->app_log->warn( 'You have selected to skip rule ' - # . $self->rule_name - # . ', but this file has changed since last your last analysis.' ); - # - # } -} - sub walk_INPUT { my $self = shift; my $ref = shift; - return if ref($ref); return unless $ref; - if ( !exists $self->files->{$ref} ) { - $self->files->{$ref} = 1; - } -} - -# memoize('compare_mtimes'); -# sub compare_mtimes { -# my $pmtime = shift; -# my $mtime = shift; -# -# my $strp = DateTime::Format::Strptime->new( -# pattern => '%a %b %e %T %Y', -# time_zone => 'local', -# ); -# -# my $dt1 = $strp->parse_datetime($pmtime); -# my $dt2 = $strp->parse_datetime($mtime); -# -# #For reasons unknown there is something off by 1 second either way -# #my $cmp = DateTime->compare( $dt1, $dt2 ); -# my $dur = $dt2->subtract_datetime($dt1); -# -# # TODO Add this formatting to HPC::Runner -# my ( $days, $hours, $minutes, $seconds ) = -# $dur->in_units( 'days', 'hours', 'minutes', 'seconds' ); -# -# if ( $days >= 1 || $hours >= 1 || $minutes >= 1 ) { -# return 1; -# } -# elsif ( $seconds >= 2 ) { -# return 1; -# } -# else { -# return 0; -# } -# } -# -# sub update_seen { -# my $self = shift; -# my $file = shift; -# my $update = shift; -# -# # $self->seen_modify->{all}->{$file} = $update; -# $self->seen_modify->{local}->{$file} = $update; -# } - -sub write_file_log { - my $self = shift; + my $ref_name = ref($ref); + return unless $ref_name; + return unless $ref_name eq 'Path::Tiny'; - my $text = ""; - my @files = keys %{ $self->seen_modify->{local} }; - @files = sort @files; - - $self->seen_modify->{local} = {}; - - if (@files) { - $text = <push_files( $ref->absolute ); } 1; diff --git a/lib/BioX/Workflow/Command/stats.pm b/lib/BioX/Workflow/Command/stats.pm index a4f961a..57c36d5 100644 --- a/lib/BioX/Workflow/Command/stats.pm +++ b/lib/BioX/Workflow/Command/stats.pm @@ -10,6 +10,7 @@ use File::Details; use File::Basename; use List::MoreUtils qw(uniq); use Try::Tiny; +use Path::Tiny; extends qw( BioX::Workflow::Command ); @@ -78,11 +79,11 @@ has 'table_log' => ( } ); -option 'use_full' => ( +option 'use_abs' => ( is => 'rw', isa => 'Bool', default => 0, - documentation => 'Use the full path name instead of the basename' + documentation => 'Use the absolute path name instead of the basename' ); our $human = Number::Bytes::Human->new( @@ -119,8 +120,8 @@ around 'pre_FILES' => sub { $self->$orig( $attr, $cond ); - for my $pair ( $self->files_pairs ) { - $self->preprocess_row( $pair->[0], $cond ); + for my $file ( $self->all_files ) { + $self->preprocess_row( $file, $cond ); } }; @@ -149,7 +150,7 @@ sub gen_row { my $rel = ''; $rel = $file; - $rel = File::Spec->abs2rel($file) if $self->use_full; + $rel = path($file)->absolute if $self->use_full; my $basename = basename($file) unless $self->use_full;