I am printing 110000 fileds one by one in CSV file but it is printing only the first result in the file
#!/usr/bin/perl
use strict;
use warnings;
use Switch;
use Text::CSV_XS;
use Data::Dumper;
use File::Slurp;
use LWP::Simple;
use Parallel::ForkManager;
my #links=("doba_media26.upcs");
my $pm = new Parallel::ForkManager(16);
foreach my $linkarray (#links) {
$pm->start and next; # do the fork
my $csv = Text::CSV_XS->new ({
binary => 1,
eol => "\r\n",
sep_char => "\t"
}) or die "Cannot use CSV: ".Text::CSV_XS->error_diag ();
open my $sample, ">:encoding(utf8)", $linkarray."_sample.csv"
or die "result.csv: $!";
my #lines = read_file( $linkarray );
my $numLines = #lines;
foreach my $line(#lines) {
eval {
my #tabs;
print $line;
push (#tabs,"");
push (#tabs,"");
push (#tabs,"");
push (#tabs,$line);
my #line = "\n";
$csv->print($sample,\#tabs);
}
};
if ($#) {
redo;
sleep(1200);
}
$pm->finish; # do the exit in the child process
}
$pm->wait_all_children;
The file has 11,000 results but the above code is printing only the first field in csv file?
Related
I'm pretty new on perl and in need for some help, basically what I want is a program that reads all .txt files from a folder, doing the script and throw the output in a new folder with a new name. Everything works when I'm working with one file at the time, specifying the name of the file.. But I can't get it to work with all of the files in the folder. This is how far I've gotten.
#!/usr/bin/perl
use warnings;
use strict;
use Path::Class;
use autodie;
use File::Find;
my #now = localtime();
my $timeStamp = sprintf(
"%04d%02d%02d-%02d:%02d:%02d",
$now[5] + 1900,
$now[4] + 1,
$now[3], $now[2], $now[1], $now[0]); #A function that translates time
my %wordcount;
my $dir = "/home/smenk/.filfolder";
opendir(DIR, $dir) || die "Kan inte öppna $dir: $!";
my #files = grep { /txt/ } readdir(DIR);
closedir DIR;
my $new_dir = dir("/home/smenk/.result"); # Reads in the folder for save
my $new_file = $new_dir->file("$timeStamp.log"); # Reads in the new file timestamp variable
open my $fh, '<', $dir or die "Kunde inte öppna '$dir' $!";
open my $fhn, '>', $new_file or die "test '$new_file'";
foreach my $file (#files) {
open(FH, "/home/smenk/.filfolder/$file") || die "Unable to open $file - $!\n";
while (<FH>) {
}
close(FH);
}
while (my $line = <$fh>) {
foreach my $str (split /\s+/, $line) {
$wordcount{$str}++;
}
}
my #listing = (sort { $wordcount{$b} <=> $wordcount{$a} } keys %wordcount)[0 .. 9];
foreach my $str (#listing) {
my $output = $wordcount{$str} . " $str\n";
print $fhn $output;
}
Here is the simplest skeleton for the reading part using Path::Class (see also dir and file:
#!/usr/bin/perl
use warnings;
use strict;
use Path::Class;
my $src = dir("/home/smenk/.filfolder");
my #txt_files = grep /[.] txt\z/x, $src->children;
for my $txt_file ( #txt_files ) {
my $in = $txt_file->openr;
while (my $line = <$in>) {
print "OUT: $line";
}
}
You can also use another great module Path::Tiny, for dir/file operations and the Time::Piece for the date/time functions - like:
#!/usr/bin/env perl
use strict;
use warnings;
use Path::Tiny;
use Time::Piece;
my #txtfiles = path("/home/smenk/.filfolder")->children(qr/\.txt\z/);
my $outdir = path("home/smenk/.result");
$outdir->mkpath; #create the dir...
my $t = localtime;
my $outfile = $outdir->child($t->strftime("%Y%m%d-%H%M%S.txt"));
$outfile->touch;
my #outdata;
for my $infile (#txtfiles) {
my #lines = $infile->lines({chomp => 1});
#do something with lines and create the output #data
push #outdata, scalar #lines;
}
$outfile->append({truncate => 1}, map { "$_\n" } #outdata); #or spew;
Here is the code:
my #col= sort keys %colnames;
print "mRNA,".join(",",#col)."\n";
foreach my $row(keys %rownames){
print "$row";
foreach my $col(#col){
my $num=$mat{$col}->{$row};
$num=~s/(\.\d\d)\d+/$1/;
print ",$num";
}
print "\n";
}
Output:
mRNA,Benzopyrene12h_replica1,Benzopyrene12h_replica2
E2F1,5.01,4.72
REV1,2.76,2.67
POLK,1.21,1.87
POLH,1.49,1.56
POLI,1.94,2.45
Please help me write this output to .csv file.
Something like this might work... Combining with Miller's answer. I didn't test it, just giving you an idea. And it's defiantly could be written more cleanly and less redundant.
use strict;
use warnings;
use autodie;
my $csvFile = Text::CSV->new ( { binary => 1, eol => "\n" } )
or die "Cannot use CSV: ".Text::CSV->error_diag ();
my #col= sort keys %colnames;
my #csv;
$csv[0][0] = "mRNA,";
my #joinCol = join(",",#col);
my $i =1;
foreach (#joinCol) {
$csv[0][$i] = $_;
$i++;
}
my $k = 1;
foreach my $row(keys %rownames){
my $j = 0;
print "$row";
$csv[$k][$j] = $row;
foreach my $col(#col){
my $num=$mat{$col}->{$row};
$num=~s/(\.\d\d)\d+/$1/;
print ",$num";
$csv[$k][$j] = $num;
$j++;
}
print "\n";
$k++;
}
open $fh, '>', "new.csv" or die "Couldn't open csv file: $! \n";
for (#csv) {
$csvFile->print($fh, $_);
}
close $fh;
To write to a CSV file, use Text::CSV
use strict;
use warnings;
use autodie;
# Your Data Initialization
my %colnames; # = Something
my %rownames; # = Something else
my %mat; # = a hash of hash
# Prepare CSV
my $csv = Text::CSV->new ( { binary => 1, eol => "\n" } )
or die "Cannot use CSV: ".Text::CSV->error_diag ();
open $fh, '>', "new.csv";
my #col = sort keys %colnames;
# Output Header
$csv->print($fh, ['mRNA', #col]);
# Output Rows
for my $row (keys %rownames){
my #data = ($row);
for my $col (#col){
my $num = $mat{$col}{$row};
$num =~ s/(\.\d\d)\d+/$1/;
push #data, $num;
}
$csv->print($fh, \#data);
}
close $fh;
I feel like I'm missing something rather obvious, but can't find any answers in the documentation. Still new to OOP with Perl, but I'm using Text::CSV to parse a CSV for later use.
How would I go about extracting the first row and pushing the values to array #headers?
Here's what I have so far:
#!/usr/bin/perl
use warnings;
use diagnostics;
use strict;
use Fcntl ':flock';
use Text::CSV;
my $csv = Text::CSV->new({ sep_char => ',' });
my $file = "sample.csv";
my #headers; # Column names
open(my $data, '<:encoding(utf8)', $file) or die "Could not open '$file' $!\n";
while (my $line = <$data>) {
chomp $line;
if ($csv->parse($line)) {
my $r = 0; # Increment row counter
my $field_count = $csv->fields(); # Number of fields in row
# While data exists...
while (my $fields = $csv->getline( $data )) {
# Parse row into columns
print "Row ".$r.": \n";
# If row zero, process headers
if($r==0) {
# Add value to #columns array
push(#headers,$fields->[$c]);
} else {
# do stuff with records...
}
}
$r++
}
close $data;
You'd think that there would be a way to reference the existing fields in the first row.
Pretty much straight from the documentation, for example.
#!/usr/bin/perl
use strict;
use warnings;
use Text::CSV_XS;
my $csv = Text::CSV_XS->new ({ binary => 1, eol => $/ });
my $file = 'o33.txt';
open my $io, "<", $file or die "$file: $!";
my $header = $csv->getline ($io);
print join("-", #$header), "\n\n";
while (my $row = $csv->getline ($io)) {
print join("-", #$row), "\n";
}
__END__
***contents of o33.txt
lastname,firstname,age,gender,phone
mcgee,bobby,27,M,555-555-5555
kincaid,marl,67,M,555-666-6666
hofhazards,duke,22,M,555-696-6969
Prints:
lastname-firstname-age-gender-phone
mcgee-bobby-27-M-555-555-5555
kincaid-marl-67-M-555-666-6666
hofhazards-duke-22-M-555-696-6969
Update: Thinking about your problem, it may be that you want to address the data by its column name. For that, you might be able to use something (also from the docs), like this:
$csv->column_names ($csv->getline ($io));
while (my $href = $csv->getline_hr ($io)) {
print "lastname is: ", $href->{lastname},
" and gender is: ", $href->{gender}, "\n"
}
Note: You can use Text::CSV instead of Text::CSV_XS, as the former is a wrapper around the latter.
Thought I'd post my results for others.
#!/usr/bin/perl
use warnings;
use diagnostics;
use strict;
use Text::CSV;
sub read_csv {
my $csv = Text::CSV->new({ sep_char => ',' });
my $file = shift;
open(my $data, '<:encoding(utf8)', $file) or die "Could not open '$file' $!\n";
# Process Row Zero
my $header = $csv->getline ($data);
my $field_count = $csv->fields();
# Read the rest of the file
while (my $line = <$data>) {
chomp $line;
# Read line if possible
if ($csv->parse($line)) {
my $r = 0;
# While data exists...
while (my $fields = $csv->getline( $data )) {
# Parse row into columns
print Display->H2;
print "Row ".$r.": ".#$fields." columns. \n";
# Print column values
for(my $c=0; $c<#$fields; $c++) {
print #$header[$c]." : ".#$fields[$c]."\n";
}
$r++
}
}
close $data;
}
}
Cheers
I have a CSV file like this:
name,email,salary
a,b#b.com,1000
d,e#e.com,2000
Now, I need to transform this to an array of hash-maps in Perl, so when I do something like:
table[1]{"email"}
it returns e#e.com.
The code I wrote is :
open(DATA, "<$file") or die "Cannot open the file\n";
my #table;
#fetch header line
$line = <DATA>;
my #header = split(',',$line);
#fetch data tuples
while($line = <DATA>)
{
my %map;
my #row = split(',',$line);
for($index = 0; $index <= $#header; $index++)
{
$map{"$header[$index]"} = $row[$index];
}
push(#table, %map);
}
close(DATA);
But I am not getting desired results.. Can u help?? Thanks in advance...
This line
push(#table, %map)
should be
push(#table, \%map)
You want table to be a list of hash references; your code adds each key and value in %map to the list as a separate element.
There is no need to reinvent the wheel here. You can do this with the Text::CSV module.
#!/usr/bin/perl
use strict;
use warnings;
use v5.16;
use Text::CSV;
my $csv = Text::CSV->new;
open my $fh, "<:encoding(utf8)", "data.csv" or die "data.csv: $!";
$csv->column_names( $csv->getline ($fh) );
while (my $row = $csv->getline_hr ($fh)) {
say $row->{email};
}
Something like this perhaps:
#!/usr/bin/perl
use strict;
use warnings;
use 5.010;
my #table;
chomp(my $header = <DATA>);
my #cols = split /,/, $header; # Should really use a real CSV parser here
while (<DATA>) {
chomp;
my %rec;
#rec{#cols} = split /,/;
push #table, \%rec;
}
say $table[1]{email};
__END__
name,email,salary
a,b#b.com,1000
d,e#e.com,2000
I have one folder that contains more number of xml files and extract some specific information form xml files. I used libxml to extract wanted information one xml and I succeded but now how can I extract from folder and each xml file using perl script. I tried like this for one xml file:
use warnings;
use strict;
use XML::LibXML::Reader;
my $file;
open( $file, 'formal.xml');
my $reader = XML::LibXML::Reader->new( IO => $file )
or die ("unable to open file");
my %hash;
while ($reader->nextElement( 'nuber' ) ) {
my $Number = $reader->readInnerXml();
$reader->nextElement( 'data' );
my $information = $reader->readOuterXml();
$nums{$Number}= $information;
print( " NUMBER:$Number\n" );
print( " Information:$information\n" );
}
print my $num=keys%hash;
close($file);
Above code working properly and extracted what I want. Now I need script that will search all files in the folder and extract the same information from all files.
use File::Find.
Your code cannot be working properly as it is. Here is an untested script that might do what you want.
use warnings; use strict;
use Carp;
use File::Find;
use File::Spec::Functions qw( canonpath );
use XML::LibXML::Reader;
die "Need directories\n" unless #ARGV;
my %hash;
find(
sub {
my $file = $_;
my $path = canonpath $File::Find::name;
return unless -f $path;
return unless $file =~ /[.]xml\z/i;
extract_information($path, \%hash);
return;
},
#ARGV
);
use Data::Dumper;
print Dumper \%hash;
sub extract_information {
my ($path, $hash) = #_;
my $ret = open my $xmlin, '<', $path;
unless ($ret) {
carp "Cannot open '$path': $!";
return;
}
my $reader = XML::LibXML::Reader->new(IO => $xmlin);
unless ($reader) {
carp "Cannot create reader using '$path'";
return;
}
while ($reader->nextElement('number')) {
my $Number = $reader->readInnerXml();
$reader->nextElement( 'data' );
my $information = $reader->readOuterXml();
$hash->{$path}{$Number} = $information;
}
close $xmlin
or carp "Cannot close '$path': $!";
return;
}