File Manager - BorneoXploit

Editing: dmarc_report.lua

--[[
Copyright (c) 2022, Vsevolod Stakhov <vsevolod@rspamd.com>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
]]--

local argparse = require "argparse"
local lua_util = require "lua_util"
local logger = require "rspamd_logger"
local lua_redis = require "lua_redis"
local lua_maps = require "lua_maps"
local dmarc_common = require "plugins/dmarc"
local rspamd_mempool = require "rspamd_mempool"
local rspamd_url = require "rspamd_url"
local rspamd_text = require "rspamd_text"
local rspamd_util = require "rspamd_util"
local rspamd_dns = require "rspamd_dns"
local fun = require "fun"

local N = 'dmarc_report'

-- Define command line options
local parser = argparse()
    :name "rspamadm dmarc_report"
    :description "Dmarc reports sending tool"
    :help_description_margin(30)

parser:option "-c --config"
      :description "Path to config file"
      :argname("<cfg>")
      :default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")

parser:flag "-v --verbose"
      :description "Enable dmarc specific logging"

parser:flag "-n --no-opt"
      :description "Do not reset reporting data/send reports"

parser:argument "date"
      :description "Date to process (yesterday by default)"
      :argname "<YYYYMMDD>"
      :args "*"
parser:option "-b --batch-size"
      :description "Send reports in batches up to <batch-size> messages"
      :argname "<number>"
      :convert(tonumber)
      :default "10"
parser:option "-w --batch-wait"
      :description "Wait <batch-wait> seconds between batches"
      :argname "<number>"
      :convert(tonumber)
      :default "0"

parser:flag "-r --recheck-rua"
      :description "Re-check RUA addresses against exclude_rua_addresses map before sending"

local report_template = [[From: "{= from_name =}" <{= from_addr =}>
To: {= rcpt =}
{%+ if is_string(bcc) %}Bcc: {= bcc =}{%- endif %}
Subject: Report Domain: {= reporting_domain =}
	Submitter: {= submitter =}
	Report-ID: {= report_id =}
Date: {= report_date =}
MIME-Version: 1.0
Message-ID: <{= message_id =}>
Content-Type: multipart/mixed;
	boundary="----=_NextPart_{= uuid =}"
Auto-Submitted: auto-generated
Precedence: bulk
X-Auto-Response-Suppress: All

This is a multipart message in MIME format.

------=_NextPart_{= uuid =}
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: 7bit

This is an aggregate report from {= submitter =}.

Report domain: {= reporting_domain =}
Submitter: {= submitter =}
Report ID: {= report_id =}

------=_NextPart_{= uuid =}
Content-Type: application/gzip
Content-Transfer-Encoding: base64
Content-Disposition: attachment;
	filename="{= submitter =}!{= reporting_domain =}!{= report_start =}!{= report_end =}.xml.gz"

]]
local report_footer = [[

------=_NextPart_{= uuid =}--]]

local dmarc_settings = {}
local redis_params
local redis_attrs = {
  config = rspamd_config,
  ev_base = rspamadm_ev_base,
  session = rspamadm_session,
  log_obj = rspamd_config,
  resolver = rspamadm_dns_resolver,
}
local redis_attrs_write = lua_util.shallowcopy(redis_attrs)
redis_attrs_write['is_write'] = true
local pool
local exclude_rua_map
-- Context for external map queries (used instead of task in rspamadm)
local map_context = {
  config = rspamd_config,
  ev_base = rspamadm_ev_base,
  session = rspamadm_session,
  resolver = rspamadm_dns_resolver,
}

local function load_config(opts)
  local _r, err = rspamd_config:load_ucl(opts['config'])

if not _r then
    logger.errx('cannot parse %s: %s', opts['config'], err)
    os.exit(1)
  end

_r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
  if not _r then
    logger.errx('cannot process %s: %s', opts['config'], err)
    os.exit(1)
  end
end

-- Concat elements using redis_keys.join_char
local function redis_prefix(...)
  return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char)
end

local function get_rua(rep_key)
  local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)

if #parts >= 3 then
    return parts[3]
  end

return nil
end

local function get_domain(rep_key)
  local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)

if #parts >= 3 then
    return parts[2]
  end

return nil
end

local function gen_uuid()
  local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
  return string.gsub(template, '[xy]', function(c)
    local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb)
    return string.format('%x', v)
  end)
end

local function gen_xml_grammar()
  local lpeg = require 'lpeg'
  local lt = lpeg.P('<') / '&lt;'
  local gt = lpeg.P('>') / '&gt;'
  local amp = lpeg.P('&') / '&amp;'
  local quot = lpeg.P('"') / '&quot;'
  local apos = lpeg.P("'") / '&apos;'
  local special = lt + gt + amp + quot + apos
  local grammar = lpeg.Cs((special + 1) ^ 0)
  return grammar
end

local xml_grammar = gen_xml_grammar()

local function escape_xml(input)
  if type(input) == 'string' or type(input) == 'userdata' then
    return xml_grammar:match(input)
  else
    input = tostring(input)

if input then
      return xml_grammar:match(input)
    end
  end

return ''
end

-- Creates report XML header
local function report_header(reporting_domain, report_start, report_end, domain_policy)
  local report_id = string.format('%s.%d.%d',
      reporting_domain, report_start, report_end)
  local xml_template = [[
<?xml version="1.0" encoding="UTF-8" ?>
<feedback>
  <report_metadata>
    <org_name>{= report_settings.org_name | escape_xml =}</org_name>
    <email>{= report_settings.email | escape_xml =}</email>
    <report_id>{= report_id =}</report_id>
    <date_range>
      <begin>{= report_start =}</begin>
      <end>{= report_end =}</end>
    </date_range>
  </report_metadata>
  <policy_published>
    <domain>{= reporting_domain | escape_xml =}</domain>
    <adkim>{= domain_policy.adkim | escape_xml =}</adkim>
    <aspf>{= domain_policy.aspf | escape_xml =}</aspf>
    <p>{= domain_policy.p | escape_xml =}</p>
    <sp>{= domain_policy.sp | escape_xml =}</sp>
    <pct>{= domain_policy.pct | escape_xml =}</pct>
  </policy_published>
]]
  return lua_util.jinja_template(xml_template, {
    report_settings = dmarc_settings.reporting,
    report_id = report_id,
    report_start = report_start,
    report_end = report_end,
    domain_policy = domain_policy,
    reporting_domain = reporting_domain,
  }, true, false,
      {
        escape_xml = escape_xml
      })
end

-- Generate xml entry for a preprocessed redis row
local function entry_to_xml(data)
  local xml_template = [[<record>
  <row>
    <source_ip>{= data.ip =}</source_ip>
    <count>{= data.count =}</count>
    <policy_evaluated>
      <disposition>{= data.disposition =}</disposition>
      <dkim>{= data.dkim_disposition =}</dkim>
      <spf>{= data.spf_disposition =}</spf>
      {% if data.override and data.override ~= '' -%}
      <reason><type>{= data.override =}</type></reason>
      {%- endif %}
    </policy_evaluated>
  </row>
  <identifiers>
    <header_from>{= data.header_from =}</header_from>
  </identifiers>
  <auth_results>
    {% if data.dkim_results[1] -%}
    {% for d in data.dkim_results -%}
    <dkim>
      <domain>{= d.domain =}</domain>
      <result>{= d.result =}</result>
    </dkim>
    {%- endfor %}
    {%- endif %}
    <spf>
      <domain>{= data.spf_domain =}</domain>
      <result>{= data.spf_result =}</result>
    </spf>
  </auth_results>
</record>
]]
  return lua_util.jinja_template(xml_template, { data = data }, true,
      false, {
        escape_xml = escape_xml
      })
end

-- Process a report entry stored in Redis splitting it to a lua table
local function process_report_entry(data, score)
  local split = lua_util.str_split(data, ',')
  local row = {
    ip = split[1],
    spf_disposition = split[2],
    dkim_disposition = split[3],
    disposition = split[4],
    override = split[5],
    header_from = split[6],
    dkim_results = {},
    spf_domain = split[11],
    spf_result = split[12],
    count = tonumber(score),
  }
  -- Process dkim entries
  local function dkim_entries_process(dkim_data, result)
    if dkim_data and dkim_data ~= '' then
      local dkim_elts = lua_util.str_split(dkim_data, '|')
      for _, d in ipairs(dkim_elts) do
        table.insert(row.dkim_results, { domain = d, result = result })
      end
    end
  end
  dkim_entries_process(split[7], 'pass')
  dkim_entries_process(split[8], 'fail')
  dkim_entries_process(split[9], 'temperror')
  dkim_entries_process(split[9], 'permerror')

return row
end

-- Process a single rua entry, validating in DNS if needed
local function process_rua(dmarc_domain, rua)
  -- Remove size limitation, as we don't care about them
  local addrs = {}
  for _, rua_part in fun.map(lua_util.str_trim, lua_util.str_split(rua, ',')) do
    local u = rspamd_url.create(pool, rua_part:gsub('!%d+[kmg]?$', ''))
    local u2 = rspamd_url.create(pool, dmarc_domain)
    if u and (u:get_protocol() or '') == 'mailto' and u:get_user() then
      -- Check each address for sanity
      if u:get_tld() == u2:get_tld() then
        -- Same eSLD - always include
        table.insert(addrs, u)
      else
        -- We need to check authority
        local resolve_str = string.format('%s._report._dmarc.%s',
            dmarc_domain, u:get_host())
        local is_ok, results = rspamd_dns.request({
          config = rspamd_config,
          session = rspamadm_session,
          type = 'txt',
          name = resolve_str,
        })

if not is_ok then
          logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part)
        else
          local found = false
          for _, t in ipairs(results) do
            if string.match(t, 'v=DMARC1') then
              found = true
              break
            end
          end

if not found then
            logger.errx('%s is not authorized to process reports on %s', dmarc_domain, u:get_host())
          else
            -- All good
            table.insert(addrs, u)
          end
        end
      end
    else
      logger.errx('invalid rua url: "%s""', rua_part)
    end
  end

if #addrs > 0 then
    return addrs
  end

return nil
end

-- Validate reporting domain, extracting rua and checking 3rd party report domains
-- This function returns a full dmarc record processed + rua as a list of url objects
local function validate_reporting_domain(reporting_domain)
  local is_ok, results = rspamd_dns.request({
    config = rspamd_config,
    session = rspamadm_session,
    type = 'txt',
    name = '_dmarc.' .. reporting_domain,
  })

if not is_ok or not results then
    logger.errx('cannot resolve _dmarc.%s: %s', reporting_domain, results)
    return nil
  end

for _, r in ipairs(results) do
    local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false)
    if processed and rec.rua then
      -- We need to check or alter rua if needed
      local processed_rua = process_rua(reporting_domain, rec.rua)
      if processed_rua then
        rec = rec.raw_elts
        rec.rua = processed_rua

-- Fill defaults in a record to avoid nils in a report
        rec['pct'] = rec['pct'] or 100
        rec['adkim'] = rec['adkim'] or 'r'
        rec['aspf'] = rec['aspf'] or 'r'
        rec['p'] = rec['p'] or 'none'
        rec['sp'] = rec['sp'] or 'none'
        return rec
      end
      return nil
    end
  end

return nil
end

-- Returns a list of recipients from a table as a string processing elements if needed
local function rcpt_list(tbl, func)
  local res = {}
  for _, r in ipairs(tbl) do
    if func then
      table.insert(res, func(r))
    else
      table.insert(res, r)
    end
  end

return table.concat(res, ',')
end

-- Synchronous smtp send function
local function send_reports_by_smtp(opts, reports, finish_cb)
  local lua_smtp = require "lua_smtp"
  local reports_failed = 0
  local reports_sent = 0
  local report_settings = dmarc_settings.reporting

local function gen_sendmail_cb(report, args)
    return function(ret, err)
      -- We modify this from all callbacks
      args.nreports = args.nreports - 1
      if not ret then
        logger.errx("Couldn't send mail for %s: %s", report.reporting_domain, err)
        reports_failed = reports_failed + 1
      else
        reports_sent = reports_sent + 1
        lua_util.debugm(N, 'successfully sent a report for %s: %s bytes sent',
            report.reporting_domain, #report.message)
      end

-- Tail call to the next batch or to the final function
      if args.nreports == 0 then
        if args.next_start > #reports then
          finish_cb(reports_sent, reports_failed)
        else
          if opts.batch_wait > 0 then
            lua_util.debugm(N, 'sleeping %d seconds before sending next batch', opts.batch_wait)
            rspamadm_ev_base:sleep(opts.batch_wait, function() args.cont_func(args.next_start) end)
          else
             args.cont_func(args.next_start)
          end
        end
      end
    end
  end

local function send_data_in_batches(cur_batch)
    local nreports = math.min(#reports - cur_batch + 1, opts.batch_size)
    local next_start = cur_batch + nreports
    lua_util.debugm(N, 'send data for %s domains (from %s to %s)',
        nreports, cur_batch, next_start - 1)
    -- Shared across all closures
    local gen_args = {
      cont_func = send_data_in_batches,
      nreports = nreports,
      next_start = next_start
    }
    for i = cur_batch, next_start - 1 do
      local report = reports[i]
      local send_opts = {
        ev_base = rspamadm_ev_base,
        session = rspamadm_session,
        config = rspamd_config,
        host = report_settings.smtp,
        port = report_settings.smtp_port or 25,
        resolver = rspamadm_dns_resolver,
        from = report_settings.email,
        recipients = report.rcpts,
        helo = report_settings.helo or 'rspamd.localhost',
      }

lua_smtp.sendmail(send_opts,
          report.message,
          gen_sendmail_cb(report, gen_args))
    end
  end

send_data_in_batches(1)
end

local function prepare_report(opts, start_time, end_time, rep_key)
  local rua = get_rua(rep_key)
  local reporting_domain = get_domain(rep_key)

if not rua then
    logger.errx('report %s has no valid rua, skip it', rep_key)
    return nil
  end
  if not reporting_domain then
    logger.errx('report %s has no valid reporting_domain, skip it', rep_key)
    return nil
  end

local ret, results = lua_redis.request(redis_params, redis_attrs,
      { 'EXISTS', rep_key })

if not ret or not results or results == 0 then
    return nil
  end

-- Rename report key to avoid races
  if not opts.no_opt then
    lua_redis.request(redis_params, redis_attrs_write,
        { 'RENAME', rep_key, rep_key .. '_processing' })
    rep_key = rep_key .. '_processing'
  end

local dmarc_record = validate_reporting_domain(reporting_domain)
  lua_util.debugm(N, 'process reporting domain %s: %s', reporting_domain, dmarc_record)

if not dmarc_record then
    if not opts.no_opt then
      lua_redis.request(redis_params, redis_attrs_write,
          { 'DEL', rep_key })
    end
    logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain)
    return nil
  end

-- Re-check RUA addresses against exclude_rua_addresses map if enabled
  -- Works with both local maps and external maps (HTTP) using synchronous requests
  if exclude_rua_map and dmarc_record.rua then
    local filtered_rua = {}
    local excluded_count = 0
    for _, rua_elt in ipairs(dmarc_record.rua) do
      local rua_email = string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host())
      -- For external maps, pass map_context to enable synchronous HTTP requests
      local excluded = exclude_rua_map:get_key(rua_email, nil, map_context)
      if not excluded then
        -- Also check just the domain part
        excluded = exclude_rua_map:get_key(rua_elt:get_host(), nil, map_context)
      end
      if excluded then
        lua_util.debugm(N, 'RUA address %s for domain %s is excluded by map (re-check)',
            rua_email, reporting_domain)
        excluded_count = excluded_count + 1
      else
        table.insert(filtered_rua, rua_elt)
      end
    end

if #filtered_rua == 0 then
      if not opts.no_opt then
        lua_redis.request(redis_params, redis_attrs_write,
            { 'DEL', rep_key })
      end
      logger.messagex('All RUA addresses for domain %s are excluded by map (re-check), skipping report',
          reporting_domain)
      return nil
    elseif excluded_count > 0 then
      logger.messagex('Filtered %s RUA addresses for domain %s, %s remaining',
          excluded_count, reporting_domain, #filtered_rua)
      dmarc_record.rua = filtered_rua
    end
  end

-- Get all reports for a domain
  ret, results = lua_redis.request(redis_params, redis_attrs,
      { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' })
  local report_entries = {}
  table.insert(report_entries,
      report_header(reporting_domain, start_time, end_time, dmarc_record))
  for i = 1, #results, 2 do
    local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1]))
    table.insert(report_entries, xml_record)
  end
  table.insert(report_entries, '</feedback>')
  local xml_to_compress = rspamd_text.fromtable(report_entries)
  lua_util.debugm(N, 'got xml: %s', xml_to_compress)

-- Prepare SMTP message
  local report_settings = dmarc_settings.reporting
  local rcpt_string = rcpt_list(dmarc_record.rua, function(rua_elt)
    return string.format('%s@%s', rua_elt:get_user(), rua_elt:get_host())
  end)
  local bcc_string
  if report_settings.bcc_addrs then
    bcc_string = rcpt_list(report_settings.bcc_addrs)
  end
  local uuid = gen_uuid()
  local rhead = lua_util.jinja_template(report_template, {
    from_name = report_settings.from_name,
    from_addr = report_settings.email,
    rcpt = rcpt_string,
    bcc = bcc_string,
    uuid = uuid,
    reporting_domain = reporting_domain,
    submitter = report_settings.domain,
    report_id = string.format('%s.%d.%d', reporting_domain, start_time,
        end_time),
    report_date = rspamd_util.time_to_string(rspamd_util.get_time()),
    message_id = rspamd_util.random_hex(16) .. '@' .. report_settings.msgid_from,
    report_start = start_time,
    report_end = end_time
  }, true,
      false, {
        escape_xml = escape_xml
      })
  local rfooter = lua_util.jinja_template(report_footer, {
    uuid = uuid,
  }, true, false, {
    escape_xml = escape_xml
  })
  local message = rspamd_text.fromtable {
    (rhead:gsub("\n", "\r\n")),
    rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73),
    rfooter:gsub("\n", "\r\n"),
  }

lua_util.debugm(N, 'got final message: %s', message)

if not opts.no_opt then
    lua_redis.request(redis_params, redis_attrs_write,
        { 'DEL', rep_key })
  end

local report_rcpts = lua_util.str_split(rcpt_string, ',')

if report_settings.bcc_addrs then
    for _, b in ipairs(report_settings.bcc_addrs) do
      table.insert(report_rcpts, b)
    end
  end

return {
    message = message,
    rcpts = report_rcpts,
    reporting_domain = reporting_domain
  }
end

local function process_report_date(opts, start_time, end_time, date)
  local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date)
  local ret, results = lua_redis.request(redis_params, redis_attrs,
      { 'EXISTS', idx_key })

if not ret or not results or results == 0 then
    logger.messagex('No reports for %s', date)
    return {}
  end

-- Rename index key to avoid races
  if not opts.no_opt then
    lua_redis.request(redis_params, redis_attrs_write,
        { 'RENAME', idx_key, idx_key .. '_processing' })
    idx_key = idx_key .. '_processing'
  end
  ret, results = lua_redis.request(redis_params, redis_attrs,
      { 'SMEMBERS', idx_key })

if not ret or not results then
    -- Remove bad key
    if not opts.no_opt then
      lua_redis.request(redis_params, redis_attrs_write,
          { 'DEL', idx_key })
    end
    logger.messagex('Cannot get reports for %s', date)
    return {}
  end

-- Process reports in batches to limit Redis connections
  local reports = {}

for batch_start = 1, #results, opts.batch_size do
    local batch_end = math.min(batch_start + opts.batch_size - 1, #results)
    lua_util.debugm(N, 'processing report batch %s to %s (of %s total)',
        batch_start, batch_end, #results)

for i = batch_start, batch_end do
      local rep = results[i]
      local report = prepare_report(opts, start_time, end_time, rep)

if report then
        table.insert(reports, report)
      end
    end

-- Force garbage collection after each batch to release Redis connections
    collectgarbage("collect")

if batch_end < #results and opts.batch_wait > 0 then
      lua_util.debugm(N, 'sleeping %d seconds before preparing next batch', opts.batch_wait)
      rspamadm_ev_base:sleep(opts.batch_wait)
    end
  end

-- Shuffle reports to make sending more fair
  lua_util.shuffle(reports)
  -- Remove processed key
  if not opts.no_opt then
    lua_redis.request(redis_params, redis_attrs_write,
        { 'DEL', idx_key })
  end

return reports
end

-- Returns a day before today at 00:00 as unix seconds
local function yesterday_midnight()
  local piecewise_time = os.date("*t")
  piecewise_time.day = piecewise_time.day - 1 -- Lua allows negative values here
  piecewise_time.hour = 0
  piecewise_time.sec = 0
  piecewise_time.min = 0
  return os.time(piecewise_time)
end

-- Returns today time at 00:00 as unix seconds
local function today_midnight()
  local piecewise_time = os.date("*t")
  piecewise_time.hour = 0
  piecewise_time.sec = 0
  piecewise_time.min = 0
  return os.time(piecewise_time)
end

local function handler(args)
  local start_time
  -- Preserve start time as report sending might take some time
  local start_collection = today_midnight()

local opts = parser:parse(args)

-- Normalize batch_size: floor to integer and clamp to >= 1
  -- Fractional values would break array indexing in batching loops
  opts.batch_size = math.max(1, math.floor(opts.batch_size or 10))
  -- Normalize batch_wait: floor to integer and clamp to >= 0
  opts.batch_wait = math.max(0, math.floor(opts.batch_wait or 0))

pool = rspamd_mempool.create()
  load_config(opts)
  rspamd_url.init(rspamd_config:get_tld_path())

if opts.verbose then
    lua_util.enable_debug_modules('dmarc', N)
  end

dmarc_settings = rspamd_config:get_all_opt('dmarc')
  if not dmarc_settings or not dmarc_settings.reporting or not dmarc_settings.reporting.enabled then
    logger.errx('dmarc reporting is not enabled, exiting')
    os.exit(1)
  end

dmarc_settings = lua_util.override_defaults(dmarc_common.default_settings, dmarc_settings)
  redis_params = lua_redis.parse_redis_server('dmarc', dmarc_settings)

if not redis_params then
    logger.errx('Redis is not configured, exiting')
    os.exit(1)
  end

-- rspamadm doesn't run worker on_load callbacks, so the async sentinel
  -- watcher in lua_redis never fires here; resolve the master synchronously
  -- before issuing any writes (see #6009).
  local setup_err
  lua_redis.prepare_redis_setup(redis_params, function(err)
    setup_err = err
  end)
  if setup_err then
    logger.errx('Cannot prepare Redis: %s', setup_err)
    os.exit(1)
  end

-- Load exclude_rua_addresses map if --recheck-rua flag is set
  if opts.recheck_rua then
    if dmarc_settings.reporting.exclude_rua_addresses then
      exclude_rua_map = lua_maps.map_add_from_ucl(dmarc_settings.reporting.exclude_rua_addresses,
          'set', 'DMARC RUA exclusion map for report sending')
      if exclude_rua_map then
        if exclude_rua_map.__external then
          logger.messagex('Loaded exclude_rua_addresses external map for RUA re-checking')
        else
          logger.messagex('Loaded exclude_rua_addresses map for RUA re-checking')
        end
      else
        logger.warnx('Failed to load exclude_rua_addresses map, RUA re-checking disabled')
      end
    else
      logger.warnx('--recheck-rua specified but no exclude_rua_addresses configured in dmarc settings')
    end
  end

for _, e in ipairs({ 'email', 'domain', 'org_name' }) do
    if not dmarc_settings.reporting[e] then
      logger.errx('Missing required setting: dmarc.reporting.%s', e)
      return
    end
  end

local ret, results = lua_redis.request(redis_params, redis_attrs, {
    'GET', 'rspamd_dmarc_last_collection'
  })

if not ret or not tonumber(results) then
    start_time = yesterday_midnight()
  else
    start_time = tonumber(results)
  end

lua_util.debugm(N, 'previous last report date is %s', start_time)

if not opts.date or #opts.date == 0 then
    opts.date = {}
    table.insert(opts.date, os.date('%Y%m%d', yesterday_midnight()))
  end

local ndates = 0
  local nreports = 0
  local all_reports = {}
  for _, date in ipairs(opts.date) do
    lua_util.debugm(N, 'Process date %s', date)
    local reports_for_date = process_report_date(opts, start_time, start_collection, date)
    if #reports_for_date > 0 then
      ndates = ndates + 1
      nreports = nreports + #reports_for_date

for _, r in ipairs(reports_for_date) do
        table.insert(all_reports, r)
      end
    end
  end

local function finish_cb(nsuccess, nfail)
    if not opts.no_opt then
      lua_util.debugm(N, 'set last report date to %s', start_collection)
      -- Hack to avoid coroutines + async functions mess: we use async redis call here
      redis_attrs_write.callback = function()
        logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
            ndates, nreports, nsuccess, nfail)
      end
      lua_redis.request(redis_params, redis_attrs_write,
          { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2,
            tostring(start_collection) })
    else
      logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
          ndates, nreports, nsuccess, nfail)
    end

pool:destroy()
  end
  if not opts.no_opt then
    send_reports_by_smtp(opts, all_reports, finish_cb)
  else
    logger.messagex('Skip sending mails due to -n / --no-opt option')
  end
end

return {
  name = 'dmarc_report',
  aliases = { 'dmarc_reporting' },
  handler = handler,
  description = parser._description
}

Cancel