Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: aggregate reports spec compliance #220

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

fix: aggregate reports spec compliance #220

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1c6b1f3
fix(report/attachment): use gzip instead of zip
cyyynthia Jun 3, 2022
049221b
fix!: add missing fields to aggregate reports
cyyynthia Jun 3, 2022
08bf3e1
style: consistent use of tabs/space in sql string
cyyynthia Jun 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions db/schema.mysql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ CREATE TABLE IF NOT EXISTS requests (
policy TINYINT NOT NULL DEFAULT '0',
spolicy TINYINT NOT NULL DEFAULT '0',
pct TINYINT NOT NULL DEFAULT '0',
fo TINYINT NOT NULL DEFAULT '0',
locked TINYINT NOT NULL DEFAULT '0',
firstseen TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
lastsent TIMESTAMP NOT NULL DEFAULT '1970-01-01 00:00:00',
Expand Down Expand Up @@ -106,6 +107,7 @@ CREATE TABLE IF NOT EXISTS messages (
from_domain INT UNSIGNED NOT NULL,
policy_domain INT UNSIGNED NOT NULL,
spf TINYINT NOT NULL,
spf_scope TINYINT NOT NULL,
align_dkim TINYINT UNSIGNED NOT NULL,
align_spf TINYINT UNSIGNED NOT NULL,
sigcount TINYINT UNSIGNED NOT NULL,
Expand Down
7 changes: 7 additions & 0 deletions opendmarc/opendmarc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2314,6 +2314,7 @@ mlfi_eom(SMFICTX *ctx)
int adkim;
int aspf;
int pct;
int fo;
int p;
int sp;
int align_dkim;
Expand Down Expand Up @@ -2856,6 +2857,9 @@ mlfi_eom(SMFICTX *ctx)
dmarcf_dstring_printf(dfc->mctx_histbuf,
"spf %d\n",
dfc->mctx_spfresult);
dmarcf_dstring_printf(dfc->mctx_histbuf,
"spf_scope %d\n",
spfmode);
wspf = TRUE;
}
else if (ar.ares_result[c].result_method == ARES_METHOD_DKIM)
Expand Down Expand Up @@ -3245,6 +3249,9 @@ mlfi_eom(SMFICTX *ctx)
opendmarc_policy_fetch_pct(cc->cctx_dmarc, &pct);
dmarcf_dstring_printf(dfc->mctx_histbuf, "pct %d\n", pct);

opendmarc_policy_fetch_fo(cc->cctx_dmarc, &fo);
dmarcf_dstring_printf(dfc->mctx_histbuf, "fo %d\n", fo);

opendmarc_policy_fetch_adkim(cc->cctx_dmarc, &adkim);
dmarcf_dstring_printf(dfc->mctx_histbuf, "adkim %d\n", adkim);

Expand Down
22 changes: 17 additions & 5 deletions reports/opendmarc-import.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ my $ipaddr;
my $jobid;
my $p;
my $pct;
my $fo;
my $pdomain;
my $policy;
my $received;
Expand All @@ -85,6 +86,7 @@ my $repuri;
my $sigcount = 0;
my $sp;
my $spf;
my $spf_scope;
my @rua;

###
Expand Down Expand Up @@ -248,16 +250,16 @@ sub update_db
$dbi_s = $dbi_h->prepare(q{
INSERT INTO messages(
date, jobid, reporter, policy, disp, ip, env_domain, from_domain,
policy_domain, spf, align_spf, align_dkim, sigcount, arc, arc_policy
policy_domain, spf, spf_scope, align_spf, align_dkim, sigcount, arc, arc_policy
)
VALUES(
FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?
)
});

if (!$dbi_s->execute(
$received, $jobid, $rep_id, $policy, $action, $ipaddr_id, $envfrom_id, $from_id,
$pdomain_id, $spf, $align_spf, $align_dkim, $sigcount, $arc, $arc_policy
$pdomain_id, $spf, $spf_scope, $align_spf, $align_dkim, $sigcount, $arc, $arc_policy
))
{
print STDERR "$progname: failed to insert message: " . $dbi_h->errstr . "\n";
Expand Down Expand Up @@ -400,9 +402,9 @@ sub update_db
$dbi_s->finish;
}

$dbi_s = $dbi_h->prepare("UPDATE requests SET adkim = ?, aspf = ?, policy = ?, spolicy = ?, pct = ? WHERE id = ?");
$dbi_s = $dbi_h->prepare("UPDATE requests SET adkim = ?, aspf = ?, policy = ?, spolicy = ?, pct = ?, fo = ? WHERE id = ?");

if (!$dbi_s->execute($adkim, $aspf, $p, $sp, $pct, $request_id))
if (!$dbi_s->execute($adkim, $aspf, $p, $sp, $pct, $fo, $request_id))
{
print STDERR "$progname: failed to update policy data for $fdomain: " . $dbi_h->errstr . "\n";
$dbi_s->finish;
Expand Down Expand Up @@ -673,6 +675,7 @@ while (<$inputfh>)
undef $jobid;
undef $p;
undef $pct;
undef $fo;
undef $pdomain;
undef $policy;
undef $received;
Expand All @@ -681,6 +684,7 @@ while (<$inputfh>)
$sigcount = 0;
undef $sp;
undef $spf;
undef $spf_scope;
}

$jobid = $value;
Expand All @@ -702,6 +706,10 @@ while (<$inputfh>)
$pct = $value;
}

case "fo" {
$fo = $value;
}

case "pdomain" {
$pdomain = $value;
}
Expand Down Expand Up @@ -733,6 +741,10 @@ while (<$inputfh>)
$spf = $value;
}

case "spf_scope" {
$spf_scope = $value;
}

else {
print STDERR "$progname: unknown key '$key' at line $lineno\n";
}
Expand Down
71 changes: 47 additions & 24 deletions reports/opendmarc-reports.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use File::Temp;
use Net::Domain qw(hostfqdn hostdomain);
use Getopt::Long;
use IO::Handle;
use IO::Compress::Zip qw(zip);
use IO::Compress::Gzip qw(gzip);
use POSIX;
use MIME::Base64;
use Net::SMTP;
Expand Down Expand Up @@ -50,7 +50,7 @@ my $boundary;
my $tmpout;

my $repfile;
my $zipfile;
my $gzipfile;

my $zipin;

Expand All @@ -70,6 +70,7 @@ my $spolicy;
my $policystr;
my $spolicystr;
my $pct;
my $fo;

my $repuri;
my @repuris;
Expand All @@ -84,9 +85,11 @@ my $align_dkimstr;
my $align_spf;
my $align_spfstr;
my $spfresult;
my $spfscope;
my $dkimresult;
my $disp;
my $spfresultstr;
my $spfscopestr;
my $dkimresultstr;
my $dispstr;
my $ipaddr;
Expand Down Expand Up @@ -447,7 +450,7 @@ foreach (@$domainset)
next;
}

$dbi_s = $dbi_h->prepare("SELECT repuri, adkim, aspf, policy, spolicy, pct, UNIX_TIMESTAMP(lastsent) FROM requests WHERE domain = ?");
$dbi_s = $dbi_h->prepare("SELECT repuri, adkim, aspf, policy, spolicy, pct, fo, UNIX_TIMESTAMP(lastsent) FROM requests WHERE domain = ?");
if (!$dbi_s->execute($domainid))
{
print STDERR "$progname: can't get reporting URI for domain $domain: " . $dbi_h->errstr . "\n";
Expand Down Expand Up @@ -486,7 +489,11 @@ foreach (@$domainset)
}
if (defined($dbi_a->[6]))
{
$lastsent = $dbi_a->[6];
$fo = $dbi_a->[6];
}
if (defined($dbi_a->[7]))
{
$lastsent = $dbi_a->[7];
}
}

Expand Down Expand Up @@ -517,7 +524,7 @@ foreach (@$domainset)

# construct the temporary file
$repfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".xml";
$zipfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".zip";
$gzipfile = $repdom . "!" . $domain . "!" . $repstart . "!" . $repend . ".xml.gz";
if (!open($tmpout, ">", $repfile))
{
print STDERR "$progname: can't create report file for domain $domain\n";
Expand Down Expand Up @@ -559,14 +566,16 @@ foreach (@$domainset)
print $tmpout "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n";
print $tmpout "<feedback>\n";

print $tmpout " <version>1</version>\n";

print $tmpout " <report_metadata>\n";
print $tmpout " <org_name>$repdom</org_name>\n";
print $tmpout " <email>$repemail</email>\n";
print $tmpout " <report_id>$domain:$now</report_id>\n";
print $tmpout " <date_range>\n";
print $tmpout " <begin>$repstart</begin>\n";
print $tmpout " <end>$repend</end>\n";
print $tmpout " </date_range>\n";
print $tmpout " <email>$repemail</email>\n";
print $tmpout " <report_id>$domain:$now</report_id>\n";
print $tmpout " <date_range>\n";
print $tmpout " <begin>$repstart</begin>\n";
print $tmpout " <end>$repend</end>\n";
print $tmpout " </date_range>\n";
print $tmpout " </report_metadata>\n";

print $tmpout " <policy_published>\n";
Expand All @@ -576,6 +585,7 @@ foreach (@$domainset)
print $tmpout " <p>$policystr</p>\n";
print $tmpout " <sp>$spolicystr</sp>\n";
print $tmpout " <pct>$pct</pct>\n";
print $tmpout " <fo>$fo</fo>\n";
print $tmpout " </policy_published>\n";

if ($daybound)
Expand All @@ -597,8 +607,8 @@ foreach (@$domainset)
{
$dbi_s = $dbi_h->prepare(q{
SELECT messages.id, ipaddr.addr, messages.disp, d1.name, d2.name,
messages.spf, messages.align_spf, messages.align_dkim,
messages.arc, messages.arc_policy
messages.spf, messages.spfscope, messages.align_spf,
messages.align_dkim, messages.arc, messages.arc_policy
FROM messages
JOIN ipaddr ON messages.ip = ipaddr.id
JOIN domains d1 ON messages.from_domain = d1.id
Expand Down Expand Up @@ -649,19 +659,23 @@ foreach (@$domainset)
}
if (defined($dbi_a->[6]))
{
$align_spf = $dbi_a->[6];
$spfscope = $dbi_a->[6];
}
if (defined($dbi_a->[7]))
{
$align_dkim = $dbi_a->[7];
$align_spf = $dbi_a->[7];
}
if (defined($dbi_a->[8]))
{
$arc = $dbi_a->[8];
$align_dkim = $dbi_a->[8];
}
if (defined($dbi_a->[9]))
{
$arcpolicy = $dbi_a->[9];
$arc = $dbi_a->[9];
}
if (defined($dbi_a->[10]))
{
$arcpolicy = $dbi_a->[10];
}

if (!defined($msgid))
Expand Down Expand Up @@ -696,6 +710,13 @@ foreach (@$domainset)
else { $spfresultstr = "unknown"; }
}

switch ($spfscope)
{
case 1 { $spfscopestr = "mfrom"; }
case 2 { $spfscopestr = "helo"; }
else { $spfscopestr = "unknown"; }
}

switch ($align_dkim)
{
case 4 { $align_dkimstr = "pass"; }
Expand Down Expand Up @@ -772,10 +793,12 @@ foreach (@$domainset)
print $tmpout " </policy_evaluated>\n";
print $tmpout " </row>\n";
print $tmpout " <identifiers>\n";
print $tmpout " <envelope_from>$envdomain</envelope_from>\n";
print $tmpout " <header_from>$fromdomain</header_from>\n";
print $tmpout " </identifiers>\n";
print $tmpout " <auth_results>\n";
print $tmpout " <spf>\n";
print $tmpout " <scope>$spfscopestr</scope>\n";
print $tmpout " <domain>$envdomain</domain>\n";
print $tmpout " <result>$spfresultstr</result>\n";
print $tmpout " </spf>\n";
Expand Down Expand Up @@ -871,9 +894,9 @@ foreach (@$domainset)
}

# zip the report
if (!zip [ $repfile ] => $zipfile)
if (!gzip [ $repfile ] => $gzipfile)
{
print STDERR "$progname: can't zip report for domain $domain: $!\n";
print STDERR "$progname: can't gzip report for domain $domain: $!\n";
next;
}

Expand Down Expand Up @@ -947,9 +970,9 @@ foreach (@$domainset)
my $datestr;
my $report_id;

if (!open($zipin, $zipfile))
if (!open($zipin, $gzipfile))
{
print STDERR "$progname: can't read zipped report for $domain: $!\n";
print STDERR "$progname: can't read gzipped report for $domain: $!\n";
next;
}

Expand Down Expand Up @@ -978,8 +1001,8 @@ foreach (@$domainset)
$mailout .= "generated at " . localtime() . "\n";
$mailout .= "\n";
$mailout .= "--$boundary\n";
$mailout .= "Content-Type: application/zip\n";
$mailout .= "Content-Disposition: attachment; filename=\"$zipfile\"\n";
$mailout .= "Content-Type: application/gzip\n";
$mailout .= "Content-Disposition: attachment; filename=\"$gzipfile\"\n";
$mailout .= "Content-Transfer-Encoding: base64\n";
$mailout .= "\n";

Expand Down Expand Up @@ -1044,7 +1067,7 @@ foreach (@$domainset)
}
}

unlink($zipfile);
unlink($gzipfile);
if (!$keepfiles)
{
unlink($repfile);
Expand Down