From b1dfcf915363bc8c704bf04c0ad91fea3738b63a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Erik=20Hedenstr=C3=B6m?= <erik@hedenstroem.com>
Date: Wed, 16 Feb 2022 01:25:51 +0100
Subject: [PATCH] Unzipping and XML parsing completed
---
cmd/subscribe.go | 96 ++++++++++++++++++++++++++++++
go.mod | 4 ++
go.sum | 6 ++
types/dmarc.go | 149 +++++++++++++++++++++++++++++++++++++++++++++++
types/email.go | 15 +++++
types/zip.go | 12 ++++
utils/zip.go | 34 +++++++++++
7 files changed, 316 insertions(+)
create mode 100644 cmd/subscribe.go
create mode 100644 types/dmarc.go
create mode 100644 types/email.go
create mode 100644 types/zip.go
create mode 100644 utils/zip.go
diff --git a/cmd/subscribe.go b/cmd/subscribe.go
new file mode 100644
index 0000000..3fa6ed2
--- /dev/null
+++ b/cmd/subscribe.go
@@ -0,0 +1,96 @@
+package cmd
+
+import (
+ "encoding/json"
+ "encoding/xml"
+ "fmt"
+ "net/url"
+ "os"
+ "os/signal"
+ "syscall"
+
+ "github.com/davecgh/go-spew/spew"
+ mqtt "github.com/eclipse/paho.mqtt.golang"
+ "github.com/spf13/cobra"
+ "github.com/spf13/viper"
+ "gitlab.hedenstroem.com/go/dmarc-prometheus-exporter/types"
+ "gitlab.hedenstroem.com/go/dmarc-prometheus-exporter/utils"
+)
+
+var subscribeCmd = &cobra.Command{
+ Use: "subscribe",
+ Short: "Subscribe to emails on an MQTT topic",
+ RunE: func(cmd *cobra.Command, args []string) error {
+
+ opts, err := createClientOptions()
+ if err != nil {
+ return err
+ }
+ client := mqtt.NewClient(opts)
+ if token := client.Connect(); token.Wait() && token.Error() != nil {
+ return token.Error()
+ }
+ client.Subscribe(viper.GetString("MQTT_TOPIC"), 0, callback)
+
+ keepAlive := make(chan os.Signal)
+ signal.Notify(keepAlive, os.Interrupt, syscall.SIGTERM)
+ <-keepAlive
+
+ return nil
+ },
+ DisableAutoGenTag: true,
+}
+
+func callback(client mqtt.Client, msg mqtt.Message) {
+ var email types.Email
+ if err := json.Unmarshal(msg.Payload(), &email); err == nil {
+ for _, attachment := range email.Attachments {
+ if attachment.ContentType == "application/zip" {
+ files, _ := utils.UnzipBytes(attachment.Content.Data)
+ for _, file := range files {
+ if file.HasSuffix(".xml") {
+ var feedback types.Feedback
+ feedback.FromFile = file.Name
+ if err := xml.Unmarshal(file.Data, &feedback); err == nil {
+ spew.Dump(feedback)
+ }
+ }
+ }
+ }
+ }
+ }
+}
+
+func createClientOptions() (*mqtt.ClientOptions, error) {
+ uri, err := url.Parse(viper.GetString("MQTT_URL"))
+ if err != nil {
+ return nil, err
+ }
+ var server string
+ switch uri.Scheme {
+ case "mqtts":
+ server = fmt.Sprintf("ssl://%s", uri.Host)
+ case "ws":
+ server = fmt.Sprintf("ws://%s", uri.Host)
+ default:
+ server = fmt.Sprintf("tcp://%s", uri.Host)
+ }
+ opts := mqtt.NewClientOptions()
+ opts.AddBroker(server)
+ if password, passwordSet := uri.User.Password(); passwordSet {
+ opts.SetUsername(uri.User.Username())
+ opts.SetPassword(password)
+ }
+ opts.SetClientID(viper.GetString("MQTT_CLIENT_ID"))
+ return opts, nil
+}
+
+func init() {
+ subscribeCmd.Flags().StringP("url", "u", "tcp://127.0.0.1:1883", "MQTT Broker URL")
+ subscribeCmd.Flags().StringP("topic", "t", "", "MQTT Topic")
+ subscribeCmd.Flags().StringP("id", "i", "dmarc-prometheus-exporter", "MQTT Client ID")
+ viper.BindPFlag("MQTT_URL", subscribeCmd.Flags().Lookup("url"))
+ viper.BindPFlag("MQTT_TOPIC", subscribeCmd.Flags().Lookup("topic"))
+ viper.BindPFlag("MQTT_CLIENT_ID", subscribeCmd.Flags().Lookup("id"))
+ RootCmd.AddCommand(subscribeCmd)
+}
diff --git a/go.mod b/go.mod
index 537868d..66e0bd7 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,8 @@ module gitlab.hedenstroem.com/go/dmarc-prometheus-exporter
go 1.17
require (
+ github.com/davecgh/go-spew v1.1.1
+ github.com/eclipse/paho.mqtt.golang v1.3.5
github.com/joho/godotenv v1.4.0
github.com/spf13/cobra v1.3.0
github.com/spf13/viper v1.10.1
@@ -11,6 +13,7 @@ require (
require (
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
+ github.com/gorilla/websocket v1.4.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/inconshreveable/mousetrap v1.0.0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
@@ -22,6 +25,7 @@ require (
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
+ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d // indirect
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
golang.org/x/text v0.3.7 // indirect
gopkg.in/ini.v1 v1.66.2 // indirect
diff --git a/go.sum b/go.sum
index ac029e5..77ea11e 100644
--- a/go.sum
+++ b/go.sum
@@ -92,6 +92,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/eclipse/paho.mqtt.golang v1.3.5 h1:sWtmgNxYM9P2sP+xEItMozsR3w0cqZFlqnNN1bdl41Y=
+github.com/eclipse/paho.mqtt.golang v1.3.5/go.mod h1:eTzb4gxwwyWpqBUHGQZ4ABAV7+Jgm1PklsYT/eo8Hcc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -191,6 +193,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M=
github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0=
@@ -428,6 +432,7 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@@ -445,6 +450,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
diff --git a/types/dmarc.go b/types/dmarc.go
new file mode 100644
index 0000000..276e7f7
--- /dev/null
+++ b/types/dmarc.go
@@ -0,0 +1,149 @@
+package types
+
+import (
+ "bytes"
+ "encoding/xml"
+ "time"
+)
+
+// Content is the structure for processing data
+type Content struct {
+ From string
+ Name string
+ Data *bytes.Buffer
+}
+
+// Row is the dmarc row in a report
+type Row struct {
+ SourceIP string
+ Count int64
+ EvalDisposition string
+ EvalSPFAlign string
+ EvalDKIMAalign string
+ Reason string
+ DKIMDomain string
+ DKIMResult string
+ SPFDomain string
+ SPFResult string
+ IdentifierHFrom string
+}
+
+// Rows is jus the report and the rows of a report
+type Rows struct {
+ Report Report
+ Rows []Row
+}
+
+// Report is the content of the report
+type Report struct {
+ ID int64
+ ReportBegin time.Time
+ ReportEnd time.Time
+ PolicyDomain string
+ ReportOrg string
+ ReportID string
+ ReportEmail string
+ ReportExtraContactInfo string
+ PolicyAdkim string
+ PolicyAspf string
+ PolicyP string
+ PolicySP string
+ PolicyPCT string
+ Count int64
+ DKIMResult string
+ SPFResult string
+ Items int
+}
+
+// Reports is the collection of reports
+type Reports struct {
+ Reports []Report
+ LastPage int
+ CurPage int
+ NextPage int
+ TotalPages int
+ Pages []int
+}
+
+type dateRange struct {
+ XMLName xml.Name `xml:"date_range"`
+ Begin int64 `xml:"begin"`
+ End int64 `xml:"end"`
+}
+
+type reportMetadata struct {
+ XMLName xml.Name `xml:"report_metadata"`
+ OrgName string `xml:"org_name"`
+ Email string `xml:"email"`
+ ExtraContactInfo string `xml:"extra_contact_info,omitempty"`
+ ReportID string `xml:"report_id"`
+ DateRange dateRange `xml:"date_range"`
+}
+
+type policyPublished struct {
+ XMLName xml.Name `xml:"policy_published"`
+ Domain string `xml:"domain"`
+ ADKIM string `xml:"adkim"`
+ ASPF string `xml:"aspf"`
+ P string `xml:"p"`
+ SP string `xml:"sp"`
+ PCT string `xml:"pct"`
+}
+
+type reason struct {
+ XMLName xml.Name `xml:"reason"`
+ Type string `xml:"type"`
+ Comment string `xml:"comment"`
+}
+
+type policyEvaluated struct {
+ XMLName xml.Name `xml:"policy_evaluated"`
+ Disposition string `xml:"disposition"`
+ DKIM string `xml:"dkim"`
+ SPF string `xml:"spf"`
+ Reasons []reason `xml:"reason"`
+}
+
+type row struct {
+ XMLName xml.Name `xml:"row"`
+ SourceIP string `xml:"source_ip"`
+ Count int64 `xml:"count"`
+ PolicyEvaluated policyEvaluated `xml:"policy_evaluated"`
+}
+
+type identify struct {
+ XMLName xml.Name `xml:"identifiers"`
+ HeaderFrom string `xml:"header_from"`
+}
+
+type spf struct {
+ XMLName xml.Name `xml:"spf"`
+ Result string `xml:"result"`
+}
+
+type dkim struct {
+ XMLName xml.Name `xml:"dkim"`
+ Result string `xml:"result"`
+}
+
+type authResult struct {
+ XMLName xml.Name `xml:"auth_results"`
+ SPF []spf `xml:"spf"`
+ DKIM []dkim `xml:"dkim"`
+}
+
+type record struct {
+ XMLName xml.Name `xml:"record"`
+ Rows []row `xml:"row"`
+ Identifiers identify `xml:"identifiers"`
+ AuthResults authResult `xml:"auth_results"`
+}
+
+// Feedback contains the reports and file information
+type Feedback struct {
+ XMLName xml.Name `xml:"feedback"`
+ FromFile string
+ ReportMetadata reportMetadata `xml:"report_metadata"`
+ PolicyPublished policyPublished `xml:"policy_published"`
+ Records []record `xml:"record"`
+}
diff --git a/types/email.go b/types/email.go
new file mode 100644
index 0000000..7ae1126
--- /dev/null
+++ b/types/email.go
@@ -0,0 +1,15 @@
+package types
+
+type Email struct {
+ Attachments []struct {
+ Type string `json:"type"`
+ ContentType string `json:"contentType"`
+ Filename string `json:"filename"`
+ Checksum string `json:"checksum"`
+ Size int `json:"size"`
+ Content struct {
+ Type string `json:"type"`
+ Data []byte `json:"data"`
+ } `json:"content"`
+ } `json:"attachments"`
+}
diff --git a/types/zip.go b/types/zip.go
new file mode 100644
index 0000000..197da19
--- /dev/null
+++ b/types/zip.go
@@ -0,0 +1,12 @@
+package types
+
+import "strings"
+
+type ZipFile struct {
+ Name string
+ Data []byte
+}
+
+func (z ZipFile) HasSuffix(suf string) bool {
+ return strings.HasSuffix(z.Name, suf)
+}
diff --git a/utils/zip.go b/utils/zip.go
new file mode 100644
index 0000000..fd60f6f
--- /dev/null
+++ b/utils/zip.go
@@ -0,0 +1,34 @@
+package utils
+
+import (
+ "archive/zip"
+ "bytes"
+ "io/ioutil"
+
+ "gitlab.hedenstroem.com/go/dmarc-prometheus-exporter/types"
+)
+
+func UnzipBytes(b []byte) ([]types.ZipFile, error) {
+ zipReader, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
+ if err != nil {
+ return nil, err
+ }
+ zfiles := make([]types.ZipFile, len(zipReader.File))
+ for i, zipFile := range zipReader.File {
+ zfiles[i].Name = zipFile.Name
+ zfiles[i].Data, err = readZipFile(zipFile)
+ if err != nil {
+ return nil, err
+ }
+ }
+ return zfiles, nil
+}
+
+func readZipFile(zf *zip.File) ([]byte, error) {
+ f, err := zf.Open()
+ if err != nil {
+ return nil, err
+ }
+ defer f.Close()
+ return ioutil.ReadAll(f)
+}
--
GitLab