Commit 3a2e718e authored by Hubert Lubaczewski's avatar Hubert Lubaczewski
Browse files

Files migrated from old SVN repo

parents
#!/usr/bin/perl
use strict;
use File::Spec;
use Data::Dumper;
use POSIX qw(strftime);
use Fatal qw(open);
#my ( $OPTIONAL_HEADER, @HEADER_ELEMENTS ) = get_optional_header_re( '%m %u@%d %p %r ' );
my ( $OPTIONAL_HEADER, @HEADER_ELEMENTS ) = get_optional_header_re( '%t (%r) [%p]: [%l-1] user=%u,db=%d ' );
my $ANALYZE_DB = shift || '*';
my $HTML_OUTPUT = 0;
my $SORT_ORDER = 'sum'; # possible: min, max, sum, count, avg
my $globals = {};
my $queries = {};
debug( 'Starting' );
normalize_data();
debug( 'Stage1 complete' );
sort_data();
debug( 'Stage2 complete' );
write_raport();
exit;
sub get_optional_header_re {
my $prefix = shift;
my %re = (
'u' => '[a-z0-9_]*',
'd' => '[a-z0-9_]*',
'r' => '(?:\d{1,3}(?:\.\d{1,3}){3}\(\d+\)|\[local\])?',
'h' => '\d{1,3}(?:\.\d{1,3}){3}|\[local\]',
'p' => '\d+',
't' => '\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d (?:[A-Z]+|\+\d\d\d\d)',
'm' => '\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d+ (?:[A-Z]+|\+\d\d\d\d)',
'l' => '\d+',
'i' => '(?:BEGIN|COMMIT|DELETE|INSERT|ROLLBACK|SELECT|SET|SHOW|UPDATE)',
'x' => '\d+',
'c' => '[a-f0-9]+\.[a-f0-9]+',
);
my @known_keys = keys %re;
my $known_re = join '|', @known_keys;
my @matched = ();
$prefix =~ s/([()\[\]])/\\$1/g;
$prefix =~ s/%($known_re)/push @matched, $1;'('.$re{$1}.')'/ge;
return $prefix, @matched;
}
sub write_raport {
print "<style>td {border: solid 1pt;}</style>\n" if $HTML_OUTPUT;
print_raport_summary();
print "<table>\n" if $HTML_OUTPUT;
print_raport_header();
for my $normal ( @{ $globals->{ 'queries_list' } } ) {
my $d = delete $queries->{ $normal };
$d->{ 'normal' } = $normal;
raport_data( $d );
}
print "</table>\n" if $HTML_OUTPUT;
return;
}
sub raport_columns {
if ( $HTML_OUTPUT ) {
print "<tr><td>" . join( "</td><td>", @_ ) . "</td></tr>\n";
}
else {
print join( "\t", @_ ) . "\n";
}
return;
}
sub print_raport_summary {
print "<table>\n" if $HTML_OUTPUT;
raport_columns( "Total-time:", sprintf "%d", $globals->{ 'time' } );
raport_columns( "Total-count:", sprintf "%d", $globals->{ 'count' } );
raport_columns( "First-header:", $globals->{ 'first_header' } ) if $globals->{ 'first_header' };
raport_columns( "Last-header:", $globals->{ 'last_header' } ) if $globals->{ 'last_header' };
print "</table>\n" if $HTML_OUTPUT;
return;
}
sub print_raport_header {
raport_columns( qw( query count min avg max totaltime count% time% factor-x fastest slowest ) );
return;
}
sub raport_data {
my $d = shift;
my @c = ();
push @c, $d->{ 'normal' };
push @c, $d->{ 'count' };
push @c, sprintf "%.2f", $d->{ 'min' };
push @c, sprintf "%.2f", $d->{ 'avg' };
push @c, sprintf "%.2f", $d->{ 'max' };
push @c, sprintf "%.2f", $d->{ 'sum' };
push @c, sprintf "%.2f", 100 * $d->{ 'count' } / $globals->{ 'count' }; # count %
push @c, sprintf "%.2f", 100 * $d->{ 'sum' } / $globals->{ 'time' }; # time %
push @c, sprintf "%.2f", ( $d->{ 'sum' } / $globals->{ 'time' } ) / ( $d->{ 'count' } / $globals->{ 'count' } ); # factor x
push @c, $d->{ 'min_sql' };
push @c, $d->{ 'max_sql' };
raport_columns( @c );
return;
}
sub sort_data {
$globals->{ 'queries_list' } = [
sort { $queries->{ $b }->{ $SORT_ORDER } <=> $queries->{ $a }->{ $SORT_ORDER } }
keys %{ $queries }
];
return;
}
sub normalize_data {
my $last = {};
my $read = 0;
while ( my $line = <STDIN> ) {
chomp $line;
$read++;
print STDERR '#' if 0 == $read % 1000;
my %prefix;
if ( $line =~ s/^($OPTIONAL_HEADER)//oi ) {
my $header = $1;
@prefix{ @HEADER_ELEMENTS } = $header =~ /$OPTIONAL_HEADER/oi;
$globals->{ 'first_header' } = $header unless $globals->{ 'first_header' };
$globals->{ 'last_header' } = $header;
}
if ( $line =~ m{ \A \s* LOG: \s+ duration: \s+ (\d+\.\d+) \s+ ms \s+ (?: statement | execute[^:]* ): \s+ (.*?) \s* \z }xms ) {
my ( $time, $sql ) = ( $1, $2 );
store_normalized_data( $last ) if $last->{ 'time' };
$last = {
'time' => $time,
'sql' => $sql,
'prefix' => \%prefix,
};
}
elsif ( $line =~ m{ \A \s* (?: LOG | NOTICE | HINT | DETAIL | WARNING | PANIC | ERROR | FATAL | CONTEXT ) : \s{1,2} }xms ) {
store_normalized_data( $last ) if $last->{ 'time' };
$last = {};
next;
}
else {
next unless $last->{ 'time' };
$last->{ 'sql' } .= ' ' . $line;
}
}
store_normalized_data( $last ) if $last->{ 'sql' };
return;
}
sub store_normalized_data {
my $d = shift;
if ( ( $ANALYZE_DB )
&& ( $ANALYZE_DB ne '*' )
&& ( $d->{ 'prefix' }->{ 'd' } ne $ANALYZE_DB ) )
{
return;
}
my $T = $d->{ 'time' };
$globals->{ 'count' }++;
$globals->{ 'time' } += $T;
my $sql = $d->{ 'sql' };
$sql =~ s/^\s*//;
$sql =~ s/\s*$//;
$sql =~ s/\s+/ /g;
my $std = lc $sql;
$std =~ s/'[^']*'/?/g;
$std =~ s/\bnull\b/?/g;
$std =~ s/\s* ( <> | >= | <= | <> | \+ | - | > | < | = ) \s* (\d+)(and|or) \s+ / $1 $2 $3 /x;
$std =~ s/\b\d+\b/?/g;
$std =~ s/ in \(\s*\$?\?(?:\s*,\s*\$?\?)*\s*\)/IN (?,,?)/gi;
$std =~ s/ \s* ( \bI?LIKE\b | <> | >= | <= | <> | \+ | - | > | < | = | ~ ) \s* / \U$1\E /gix;
$std =~ s/\s*;\s*$//;
$std =~ s/\s+ = \s+ - \s+ \? \s*/ = ? /gx;
if ( $std =~ s{ \A declare \s+ (\S+) \s+ (.*?) \s+ for \s+ (.*) \z }{declare ? $2 for $3}xms ) {
$globals->{ 'cursor' }->{ $1 } = $3;
}
$std =~ s{ \A (fetch \s+ .*? \s+ from ) \s+ (\S+) \s* \z}{$1 [$globals->{'cursor'}->{$2}]}xms;
$std =~ s{ \A prepare\s+ [^\s(]+ }{prepare STATEMENT}xms;
$std =~ s{ \A deallocate \s+ \S+ }{deallocate STATEMENT}xms;
$std =~ s{ \A (?: abort | rollback | rollback \s+ transaction ) \s* \z }{rollback}xms;
$std =~ s{ \A (?: end | commit | commit \s+ transaction ) \s* \z }{commit}xms;
$std =~ s{ \A (?: begin | begin \s+ work | begin \s+ transaction | start \s+ transaction ) \s* \z }{begin}xms;
$std =~ s/--.*$//mg;
$std =~ s{/\*.*?\*/}{}sg;
$std =~ s/^\s*//;
$std =~ s/\s*$//;
$std =~ s/\s+/ /g;
$std =~ s/\?+/?/g;
my $ID = qr{
(?:
[a-z_0-9]+
|
" (?: [^"]* | "" )+ "
)
}ix;
my $FIELD = qr{
(?:
(?:
$ID(?:\.$ID)?
)
(?:
\s*
[<>=]+
\s*
(?: $ID | \? )
)?
(?:
\s+
as
\s+
$ID
)?
)
}ixo;
$std =~ s/select $FIELD(?:\s*,\s*$FIELD)* FROM/select SOME_FIELDS from/goi;
$queries->{ $std } ||= {};
my $Q = $queries->{ $std };
$Q->{ 'count' }++;
$Q->{ 'sum' } += $T;
$Q->{ 'avg' } = $Q->{ 'sum' } / $Q->{ 'count' };
@$Q{ qw(min min_sql) } = ( $T, $sql ) if !$Q->{ 'min' } || $Q->{ 'min' } > $T;
@$Q{ qw(max max_sql) } = ( $T, $sql ) if !$Q->{ 'max' } || $Q->{ 'max' } < $T;
return;
}
sub debug {
my $what = shift;
printf STDERR '[%s] %s%s', strftime( '%Y-%m-%d %H:%M:%S', localtime time ), $what, "\n";
return;
}
#!/usr/bin/perl
use strict;
use File::Spec;
use Data::Dumper;
use POSIX qw(strftime);
use Fatal qw(open);
use bignum;
use HTTP::Date;
my $MINIMAL_IDLE_TO_REPORT = 100;
my ( $OPTIONAL_HEADER, @HEADER_ELEMENTS ) = get_optional_header_re( '%m %u@%d %p %r ' );
my $ANALYZE_DB = shift || '*';
my $COMMIT_SQL = qr{(?:COMMIT|ROLLBACK|END|ABORT)}i;
my $BEGIN_SQL = qr{(?:BEGIN|START)}i;
my %pids = ();
parse_stdin();
exit;
sub get_optional_header_re {
my $prefix = shift;
my %re = (
'u' => '[a-z0-9_]+',
'd' => '[a-z0-9_]+',
'r' => '\d{1,3}(?:\.\d{1,3}){3}\(\d+\)|\[local\]',
'h' => '\d{1,3}(?:\.\d{1,3}){3}|\[local\]',
'p' => '\d+',
't' => '\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d (?:[A-Z]+|\+\d\d\d\d)',
'm' => '\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d+ (?:[A-Z]+|\+\d\d\d\d)',
'l' => '[A-Z]+',
'i' => '(?:BEGIN|COMMIT|DELETE|INSERT|ROLLBACK|SELECT|SET|SHOW|UPDATE)',
);
my @known_keys = keys %re;
my $known_re = join '|', @known_keys;
my @matched = ();
$prefix =~ s/([()\[\]])/\\$1/g;
$prefix =~ s/%($known_re)/push @matched, $1;'('.$re{$1}.')'/ge;
return $prefix, @matched;
}
sub parse_stdin {
my $last = {};
my $read = 0;
while ( my $line = <STDIN> ) {
chomp $line;
$read++;
print STDERR '#' if 0 == $read % 1000;
my @temp = $line =~ m/^$OPTIONAL_HEADER/i;
$line =~ s/^$OPTIONAL_HEADER//i;
my %prefix;
@prefix{ @HEADER_ELEMENTS } = @temp;
if ( $line =~ m{ \A \s* LOG: \s+ duration: \s+ (\d+\.\d+) \s+ ms \s+ (?: statement | execute[^:]* ): \s+ (.*?) \s* \z }xms ) {
my ( $time, $sql ) = ( $1, $2 );
process_query( $last ) if $last->{ 'time' };
$last = {
'time' => $time,
'sql' => $sql,
'prefix' => \%prefix,
};
}
elsif ( $line =~ m{ \A \s* (?: LOG | NOTICE | HINT | DETAIL | WARNING | PANIC | ERROR ) : \s{1,2} }xms ) {
process_query( $last ) if $last->{ 'time' };
$last = {};
next;
}
else {
next unless $last->{ 'time' };
$last->{ 'sql' } .= ' ' . $line;
}
}
process_query( $last ) if $last->{ 'sql' };
return;
}
sub process_query {
my $d = shift;
if ( ( $ANALYZE_DB )
&& ( $ANALYZE_DB ne '*' )
&& ( $d->{ 'prefix' }->{ 'd' } ne $ANALYZE_DB ) )
{
return;
}
return unless $d->{ 'prefix' }->{ 'm' };
return unless $d->{ 'prefix' }->{ 'p' };
my $end_time = get_ms_from_timestamp( $d->{ 'prefix' }->{ 'm' } );
return unless $end_time;
my $beginning_time = $end_time - $d->{ 'time' };
my $pid = $d->{ 'prefix' }->{ 'p' };
if ( $pids{ $pid } ) {
my $previous_end = $pids{ $pid }->{ 'last' };
if ( $beginning_time - $previous_end >= $MINIMAL_IDLE_TO_REPORT ) {
printf( "%ums @ %s (pid: %s), queries:\n", $beginning_time - $previous_end, $d->{ 'prefix' }->{ 'm' }, $pid );
print " - $_\n" for ( @{ $pids{ $pid }->{ 'queries' } }, $d->{ 'sql' } );
print "\n";
}
}
if ( $d->{ 'sql' } =~ m{\A\s*$COMMIT_SQL\s*;?\s*\z} ) {
delete $pids{ $pid };
return;
}
my $qr = qr{\A\s*(?:$COMMIT_SQL\s*;\s*)?$BEGIN_SQL\s*;?\s*\z};
if ( $d->{ 'sql' } =~ m{\A\s*(?:$COMMIT_SQL\s*;\s*)?$BEGIN_SQL\s*;?\s*\z} ) {
$pids{ $pid } = {
'last' => $end_time,
'queries' => [ $d->{ 'sql' } ],
};
return;
}
if ( $pids{ $pid } ) {
$pids{ $pid }->{ 'last' } = $end_time;
push @{ $pids{ $pid }->{ 'queries' } }, $d->{ 'sql' };
}
return;
}
sub get_ms_from_timestamp {
my $timestamp = shift;
return unless $timestamp =~ m{\A(\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d(?:\.\d{1,3})?)};
return 1000 * str2time( $1 );
}
#!/usr/bin/env perl
use strict;
use DBI;
use Data::Dumper;
my @db_conn_data = @ARGV;
our @USE_SCHEMAS = qw( public );
my %cant_be_together = (
'public.posts' => 'public.topics',
'public.topics' => 'public.posts',
'public.forums' => 'public.posts',
'public.galleries' => 'public.photos',
'public.photos' => 'public.galleries',
);
my %tablespaces = ();
my $indexes_for = {};
my $size_of = {};
TRY:
while (1) {
my $data = get_data_from_db();
%tablespaces = ();
printf "%-6s %-42s ||| %-42s ||| %s\n", '', "tablespace: primary", "tablespace: secondary", "score";
printf "%-6s %s+++%s+++%s\n", '', ("-"x(42+2))x2, "-"x10;
printf '%-6s %6s | %-15s | %-15s ||| %6s | %-15s | %-15s ||| score' . "\n", '', 'tables', 'idx_tup_fetch', 'writes', 'tables', 'idx_tup_fetch', 'writes', ;
printf '%-6s %8s+%17s+%17s+++%8s+%17s+%17s+++%s' . "\n", '', map { "-"x$_ } qw(8 17 17 8 17 17 10) ;
for my $table_name (keys %{ $data }) {
my $table = $data->{ $table_name };
my @possible_tablespaces = get_possible_tablespaces($table_name);
my $tbspc_name = $possible_tablespaces[rand @possible_tablespaces];
add_table_to_tablespace($table, $tbspc_name);
}
print_tablespaces_status('------');
my $i = 0;
while (1) {
my $current_result = get_result();
my @d_tables = grep { !$cant_be_together{$_} } keys %{ $tablespaces{'primary'}->{'tables'} };
my @l_tables = grep { !$cant_be_together{$_} } keys %{ $tablespaces{'secondary'}->{'tables'} };
my $primary_table = $d_tables[rand @d_tables];
my $secondary_table = $l_tables[rand @d_tables];
my $dto = delete_table_from_tablespace($primary_table, 'primary');
my $lto = delete_table_from_tablespace($secondary_table, 'secondary');
add_table_to_tablespace($dto, 'secondary');
add_table_to_tablespace($lto, 'primary');
my $new_result = get_result();
if ($new_result < $current_result) {
delete_table_from_tablespace($primary_table, 'secondary');
delete_table_from_tablespace($secondary_table, 'primary');
add_table_to_tablespace($dto, 'primary');
add_table_to_tablespace($lto, 'secondary');
}
$i++;
if (0 == $i % 1000) {
print_tablespaces_status($i);
open my $fh, ">", "tablespace.split.out" or die "oops?!";
print_tablespaces_tables($fh);
close $fh;
exit if $new_result >= 1.9999;
next TRY if $i > 20000;
}
}
}
exit;
sub print_tablespaces_tables {
my $fh = shift;
my $tb_sizes = {};
for my $t (qw(primary secondary)) {
my $index_tablespace = ( $t eq 'primary' ? 'secondary' : 'primary' );
my @ts = sort grep { $_ } keys %{$tablespaces{$t}->{'tables'}};
for my $table (@ts) {
$tb_sizes->{$t} += $size_of->{$table};
printf $fh ("ALTER TABLE %-60s SET TABLESPACE %s;\n", $table, $t);
for my $index (@{ $indexes_for->{ $table } }) {
$tb_sizes->{$index_tablespace} += $size_of->{$index};
printf $fh ("ALTER INDEX %-60s SET TABLESPACE %s;\n", $index, $index_tablespace);
}
}
}
for my $t (qw(primary secondary)) {
printf $fh "-- Total size of %-15s : %s\n", $t, $tb_sizes->{$t};
}
return;
}
sub get_possible_tablespaces {
my $table = shift;
my $bad = $cant_be_together{$table};
return ('primary', 'secondary') unless $bad;
if ($tablespaces{'primary'}->{'tables'}->{$bad}) {
return 'secondary';
}
if ($tablespaces{'secondary'}->{'tables'}->{$bad}) {
return 'primary';
}
return ('primary', 'secondary');
}
sub add_table_to_tablespace {
my ($table, $tablespace_name) = @_;
my $T = $tablespaces{ $tablespace_name } ||= {};
$T->{'tables'}->{ $table->{'fullname'} } = $table;
for my $i (qw(idx_scans writes idx_tup_fetch)) {
$T->{$i} += $table->{ $i };
}
return;
}
sub delete_table_from_tablespace {
my ($tn, $tspace) = @_;
my $T = $tablespaces{ $tspace};
my $table = $T->{'tables'}->{$tn};
delete $T->{'tables'}->{$tn};
for my $i (qw(idx_scans writes idx_tup_fetch)) {
$T->{$i} -= $table->{ $i };
}
return $table;
}
sub get_result {
my $idx_pct = 0;
if ($tablespaces{'primary'}->{'idx_tup_fetch'} > $tablespaces{'secondary'}->{ 'idx_tup_fetch'}) {
$idx_pct = $tablespaces{'secondary'}->{ 'idx_tup_fetch'} / $tablespaces{'primary'}->{'idx_tup_fetch'}
} else {
$idx_pct = $tablespaces{'primary'}->{ 'idx_tup_fetch'} / $tablespaces{'secondary'}->{'idx_tup_fetch'}
}
my $write_pct = 0;
if ($tablespaces{'primary'}->{'writes'} > $tablespaces{'secondary'}->{ 'writes'}) {
$write_pct = $tablespaces{'secondary'}->{ 'writes'} / $tablespaces{'primary'}->{'writes'}
} else {
$write_pct = $tablespaces{'primary'}->{ 'writes'} / $tablespaces{'secondary'}->{'writes'}
}
return $idx_pct + $write_pct;
}
sub print_tablespaces_status {
my $key = shift;
printf '%6s: %6s | %15s | %15s ||| %6s | %15s | %15s ||| %8.6f' . "\n",
$key,
scalar keys %{$tablespaces{'primary'}->{'tables'}},
_f($tablespaces{'primary'}->{ 'idx_tup_fetch'}),
_f($tablespaces{'primary'}->{ 'writes'}),
scalar keys %{$tablespaces{'secondary'}->{'tables'}},
_f($tablespaces{'secondary'}->{ 'idx_tup_fetch'}),
_f($tablespaces{'secondary'}->{ 'writes'}),
get_result(),
;
return;
}
sub _f {
my $x = scalar reverse shift;
$x =~ s/(\d\d\d)(?=\d)/$1 /g;
return scalar reverse $x;
}
sub get_db_connection {
return DBI->connect(
@db_conn_data[0, 1, 2],
{
'AutoCommit' => 1,
'PrintError' => 1,
'RaiseError' => 1,
'pg_server_prepare' => 0,
}
);
}
sub get_data_from_db {
my $dbh = get_db_connection();
my $schema_question_marks = join ',', map { '?' } @USE_SCHEMAS;
my $data = $dbh->selectall_hashref(
'SELECT schemaname || ? || relname as fullname, greatest(coalesce(idx_tup_fetch, 0), 0.001) as idx_tup_fetch, greatest( coalesce( n_tup_del + n_tup_ins + n_tup_upd, 0), 0.001) as writes FROM pg_stat_user_tables where schemaname in ' . "( $schema_question_marks )",
'fullname',
undef,
'.',
@USE_SCHEMAS,
);
my $indexes = {};
my $sth = $dbh->prepare(
'SELECT schemaname || ? || tablename as tablename, schemaname || ? || indexname as indexname FROM pg_indexes',
);
$sth->execute(qw( . . ) );
while (my $temp = $sth->fetchrow_hashref()) {
push @{ $indexes->{ $temp->{'tablename'} } }, $temp->{'indexname'};
}
$sth->finish;
$size_of = $dbh->selectall_hashref(
'SELECT n.nspname || ? || c.relname as fullname, pg_relation_size(c.oid) as size, c.relkind
FROM pg_class c join pg_namespace n on c.relnamespace = n.oid
WHERE c.relkind ~ ? and n.nspname in ' . "( $schema_question_marks )",
'fullname',
undef,
'.',
'^[ri]$',
@USE_SCHEMAS,
);
$dbh->disconnect();
$indexes_for = $indexes;