diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1901796
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+osel
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..60c84d4
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,18 @@
+# Makefile for the 'osel' project.
+
+# Note that the installation of go and vgo is accomplished by
+# tools/test-setup.sh
+
+SOURCE?=./...
+
+env:
+ @echo "Running build"
+ $(HOME)/go/bin/vgo build
+
+test:
+ @echo "Running tests"
+ $(HOME)/go/bin/vgo test $(SOURCE) -cover
+
+fmt:
+ @echo "Running fmt"
+ go fmt $(SOURCE)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..8fade1f
--- /dev/null
+++ b/README.md
@@ -0,0 +1,89 @@
+OpenStack Event Listener
+========================
+
+What does this do?
+------------------
+
+The OpenStack Event Listener connects to the OpenStack message bus (RabbitMQ)
+and listens for certain kinds of events. When it detects those events, it will
+gather additional data and forward the information to external systems for
+processing. It integrates with syslog and the Qualys API.
+
+The initial use case that inspired this project was to detect when security
+group changes occurred and to trigger an external port scan of the affected IP
+addresses so that we could ensure that the change did not create a new
+vulnerability by opening something up to the Internet.
+
+For more background information on this project, see [the story of
+osel](STORY.md).
+
+Current State
+-------------
+Code maturity is considered experimental.
+
+Installation
+------------
+Use `go get git.openstack.org/openstack/osel`. Or alternatively,
+download or clone the repository.
+
+The lib was developed and tested on go 1.10.
+
+Configuration
+-------------
+
+Configuration resides in a YAML-format configuration file. Before running the
+os_event_listener process set the EL_CONFIG environment variable to the
+absolute path to that file.
+
+This is an example of the configuration format:
+
+```yaml
+debug: true
+batch_interval: 2
+rabbit_uri: "amqp://amqp_user:amqp_password@amqp_host:amqp_port//"
+logfile: "/var/log/os_event_listener.log"
+syslog_server: your.syslog.server.fqdn
+syslog_port: "514"
+syslog_protocol: "tcp"
+retry_syslog: "false"
+openstack:
+ identity_endpoint: "https://keystone.url:5000/v2.0/"
+ tenant_name: "tenant_to_authenticate_against"
+ user: "username"
+ password: "password"
+ region: "region_name"
+qualys:
+ username: "qualys_username"
+ password: "qualys_password"
+ option: "Name Of The Qualys Scan Profile"
+ proxy_url: "http://in.case.you.need.to.proxy.to.reach.qualys/"
+ url: "https://qualysapi.qualys.com/api/2.0/fo/scan/"
+ drop6: true
+```
+
+Testing
+-------
+There is one type of test file. The `*_test.go` are standard golang unit test
+files. The examples can be run as integration tests.
+
+License
+-------
+Apache v2.
+
+Contributing
+------------
+The code repository utilizes the OpenStack CI infrastructure. Please use the
+[recommended
+workflow](http://docs.openstack.org/infra/manual/developers.html#development-workflow).
+If you are not a member yet, please consider joining as an [OpenStack
+contributor](http://docs.openstack.org/infra/manual/developers.html). If you
+have questions or comments, you can email the maintainer(s).
+
+Coding Style
+------------
+The source code is automatically formatted to follow `go fmt`.
+
+OpenStack Environment
+---------------------
+* Release note management is done using [reno](https://docs.openstack.org/reno/latest/user/usage.html)
+* Zuul CI jobs are defined in-repo, [using these techniques](https://docs.openstack.org/infra/manual/zuulv3.html#howto-in-repo)
diff --git a/STORY.md b/STORY.md
new file mode 100644
index 0000000..89fc125
--- /dev/null
+++ b/STORY.md
@@ -0,0 +1,141 @@
+# OpenStack Event Listener
+
+## What is STORY.md?
+
+Retweeted Safia Abdalla (@captainsafia):
+
+From now on, you can expect to see a "STORY.md" in each of my @github repos
+that describes the technical story/process behind the project.
+
+https://twitter.com/captainsafia/status/839587421247389696
+
+## Introduction
+
+I wanted to write a little about a project that I enjoyed working on, called
+the OpenStack Event Listener, or "OSEL" for short. This project bridged the
+OpenStack control plane on one hand, and an external scanning facility provided
+by Qualys. It had a number of interesting challenges. I was never able to
+really concentrate on it - this project took about 20% of my time for a period
+of about 3 months.
+
+I am writing this partially as catharsis, to allow my brain to mark this part
+of my mental inventory as ripe for reclamation. I am also writing on the off
+chance that someone might find this useful in the future.
+
+## The Setting
+
+Let me paint a picture of the environment in which this development occurred.
+
+The Comcast OpenStack environment was transitioning from the OpenStack Icehouse
+release (very old) to the Newton release (much more current). This development
+occurred within the context of the Icehouse environment.
+
+Comcast's security team uses S3 RiskFabric to manage auditing and tracking
+security vulnerabilities across the board. They also engage the services of
+Qualys to perform network scanning (in a manner very similar to Nessus) once a
+day against all the CIDR blocks that comprise Comcast's Internet-routable IP
+addresses. Qualys scanning could also be triggered on-demand.
+
+## Technical Requirements
+
+First, let me describe the technical requirements for OSEL:
+
+* OSEL would connect to the OpenStack RabbitMQ message bus and register as a
+ listener for "notification" events. This would allow OSEL to inspect all
+ events, including security group changes.
+* When a security group change occurred, OSEL would ensure that it had the
+ details of the change (ports permitted or blocked) as well as a list of all
+ affected IP addresses.
+* OSEL would initiate a Qualys scan using the Qualys API. This would return a
+ scan ID.
+* OSEL would log the change as well as the Qualys scan ID to the Security
+ instance of Splunk to create an audit trail.
+* Qualys scan results would be imported into S3 RiskFabric for security audit
+ management.
+
+## Implementation Approach
+
+My group does most of it's development in Go, and this was no exception.
+
+This is what the data I was getting back from the AMQP message looked like.
+All identifiers have been scrambled.
+
+```json
+{
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-f96ea9a5-435e-4177-8e51-bfe60d0fae2a",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 18:10:59.112712",
+ "_context_tenant_id":"ada3b9b06482909f9361e803b54f5f32",
+ "_unique_id":"eafc9362327442b49d8c03b0e88d0216",
+ "_context_tenant_name":"BLURP",
+ "_context_user":"bca89c1b248e4a78282899ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4a78282899ece9e744cc54",
+ "payload":{
+ "security_group_rule_id":"bf8318fc-f9cb-446b-ffae-a8de016c562"
+ },
+ "_context_project_name":"BLURP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b06482909f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b06482909f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 18:10:59.079179",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl1",
+ "message_id":"e75fb2ee-85bf-44ba-a083-2445eca2ae10"
+}
+```
+
+## Testing Pattern
+
+I leaned heavily on dependency injection to make this code as testable as
+possible. For example, I needed an object that would contain the persistent
+`syslog.Writer`. I created a `SyslogActioner` interface to represent all
+interactions with syslog. When the code is operating normally, interactions
+with syslog occur through methods of the `SyslogActions` struct. But in unit
+testing mode the `SyslogTestActions` struct is used instead, and all that does
+is save copies of all messages that would have been sent so they can be
+compared against the intended messages. This facilitates good testing.
+
+## Fate of the Project
+
+The OSEL project was implemented and installed into production. There were two
+problems with it.
+
+The first to become visible is that there was no exponential backoff for the
+AMQP connection to the OpenStack control plane's RabbitMQ. When that RabbitMQ
+had issues - which was surprisingly often - OSEL would hanner away, trying to
+connect to it. That would not be too much of an issue; despite what was
+effectively an infinite loop, CPU usage was not extreme. The real problem was
+that connection failures were logged - and logs could become several gigabytes
+in a matter of hours. This was mitigated by the OpenStack operations team
+rotating the logs hourly, and alerting if an hour's worth of logs exceeded a
+set size. It was my intention to use one of the many [exponential backoff
+modules](https://github.com/cenkalti/backoff) available out there to make this
+more graceful.
+
+The second - and fatal - issue is that S3 RiskFabric was not configured to
+ingest from Qualys scans more than once a day. Since Qualys was already
+scanning the CIDR block that corresponded to our OpenStack instances once a
+day, we were essentially just adding noise to the system. The frequency of the
+S3-Qualys imports could not be easily altered, and as a result the project was
+shelved.
+
+## Remaining Work
+
+If OSEL were ever to be un-shelved, here are a few of the things that I wish I
+had time to implement.
+
+- Neutron Port Events: The initial release of OSEL processed only security
+ group rule additions, modifications, or deletions. So that covered the base
+ case for when a security group was already associated with a set of OpenStack
+ Networking (neutron) ports. But a scan should be similarly launched when a
+ new port is created and associated to a security group. This is what happens
+ when a new host is created.
+- Modern OpenStack: In order to make this work with a more modern OpenStack, it
+ would probably best to integrate with events generated through Aodh. Aodh
+ seems to be built for this kind of reporting.
+- Implement exponential backoff for AMQP connections as mentioned earlier.
diff --git a/amqp.go b/amqp.go
new file mode 100644
index 0000000..bcd600e
--- /dev/null
+++ b/amqp.go
@@ -0,0 +1,92 @@
+package main
+
+/*
+
+amqp - This file includes all of the logic necessary to interact with the amqp
+library. This is extrapolated out so that a AmqpInterface interface can be
+passed to functions. Doing this allows testing by mock classes to be created
+that can be passed to functions.
+
+Since this is a wrapper around the amqp library, this does not need testing.
+
+*/
+
+import (
+ "fmt"
+ "log"
+
+ "github.com/streadway/amqp"
+)
+
+// AmqpActioner is an interface for an AmqpActions class. Having
+// this as an interface allows us to pass in a dummy class for testing that
+// just returns mocked data.
+type AmqpActioner interface {
+ Connect() (<-chan amqp.Delivery, error)
+}
+
+// AmqpActions is a class that handles all interactions directly with Amqp.
+// See the comment on AmqpActioner for rationale.
+type AmqpActions struct {
+ Incoming *<-chan amqp.Delivery
+ Options AmqpOptions
+ AmqpConnection *amqp.Connection
+ AmqpChannel *amqp.Channel
+ NotifyError chan *amqp.Error
+}
+
+// AmqpOptions is a class to convey all of the configurable options for the
+// AmqpActions class.
+type AmqpOptions struct {
+ RabbitURI string
+}
+
+// Connect initiates the initial connection to the AMQP.
+func (s *AmqpActions) Connect() (<-chan amqp.Delivery, chan *amqp.Error, error) {
+ var err error
+
+ s.AmqpConnection, err = amqp.Dial(s.Options.RabbitURI)
+ if err != nil {
+ return nil, nil, fmt.Errorf("Failed to connect to RabbitMQ: %s", err)
+ }
+ s.NotifyError = s.AmqpConnection.NotifyClose(make(chan *amqp.Error)) //error channel
+
+ s.AmqpChannel, err = s.AmqpConnection.Channel()
+ if err != nil {
+ return nil, nil, fmt.Errorf("Failed to open a channel: %s", err)
+ }
+
+ amqpQueue, err := s.AmqpChannel.QueueDeclare(
+ "notifications.info", // name
+ false, // durable
+ false, // delete when usused
+ false, // exclusive
+ false, // no-wait
+ nil, // arguments
+ )
+ if err != nil {
+ return nil, nil, fmt.Errorf("Failed to declare a queue: %s", err)
+ }
+
+ amqpIncoming, err := s.AmqpChannel.Consume(
+ amqpQueue.Name, // queue
+ "osel", // consumer
+ true, // auto-ack
+ false, // exclusive
+ false, // no-local
+ false, // no-wait
+ nil, // args
+ )
+ if err != nil {
+ return nil, nil, fmt.Errorf("Failed to register a consumer: %s", err)
+ }
+ s.Incoming = &amqpIncoming
+ return amqpIncoming, s.NotifyError, nil
+}
+
+// Close closes connections
+func (s AmqpActions) Close() {
+ log.Println("Closing AMQP connection")
+ s.AmqpConnection.Close()
+ s.AmqpChannel.Close()
+}
diff --git a/bindep.txt b/bindep.txt
new file mode 100644
index 0000000..ae5c1bb
--- /dev/null
+++ b/bindep.txt
@@ -0,0 +1,3 @@
+build-essential
+make
+clang
diff --git a/events.go b/events.go
new file mode 100644
index 0000000..02ffab9
--- /dev/null
+++ b/events.go
@@ -0,0 +1,57 @@
+package main
+
+import (
+ "encoding/json"
+ "log"
+ "strings"
+)
+
+// EventProcessor is an Interface for event-specific classes that will process
+// events based on their specific fiends.
+type EventProcessor interface {
+ FormatLogs(*Event, []string) ([]string, error)
+ FillExtraData(*Event, OpenStackActioner) error
+}
+
+// Event is a class representing an event accepted from the AMQP, and the
+// additional attributes that have been parsed from it.
+type Event struct {
+ EventData *openStackEvent
+ RawData []byte
+ IPs map[string][]string
+ SecurityGroupRules []*osSecurityGroupRule
+ LogLines []string
+ Processor EventProcessor
+ QualysScanID string
+ QualysScanError string
+}
+
+// ParseEvent takes the []byte that has been received from the AMQP message,
+// demarshals the JSON, and then returns the event data as well as an event
+// processor specific to that type of event.
+func ParseEvent(message []byte) (Event, error) {
+ var osEvent openStackEvent
+ if err := json.Unmarshal(message, &osEvent); err != nil {
+ return Event{}, err
+ }
+
+ e := Event{
+ EventData: &osEvent,
+ RawData: message,
+ }
+
+ if Debug {
+ log.Printf("Event detected: %s\n", osEvent.EventType)
+ }
+
+ switch {
+ case strings.Contains(e.EventData.EventType, "security_group_rule.create.end"):
+ e.Processor = EventSecurityGroupRuleChange{ChangeType: "sg_rule_add"}
+ case strings.Contains(e.EventData.EventType, "security_group_rule.delete.end"):
+ e.Processor = EventSecurityGroupRuleChange{ChangeType: "sg_rule_del"}
+ // case strings.Contains(e.EventData.EventType, "port.create.end"):
+ // e.Processor = EventPortChange{ChangeType: "port_create"}
+ }
+
+ return e, nil
+}
diff --git a/events_json_fixtures_test.go b/events_json_fixtures_test.go
new file mode 100644
index 0000000..f8574d7
--- /dev/null
+++ b/events_json_fixtures_test.go
@@ -0,0 +1,299 @@
+package main
+
+const (
+ portCreateWhenCreatingInstance = `
+ {
+ "_context_roles": [
+ "admin"
+ ],
+ "_context_request_id": "req-fdb23f2e-9c0e-46b1-802f-3194c1fad251",
+ "event_type": "port.create.end",
+ "timestamp": "2016-10-03 18:40:34.596836",
+ "_context_tenant_id": "0b65cf220eab4a3cbd68681d188d7dc7",
+ "_unique_id": "bca88f14c46e40559e981ac0b4ffebf5",
+ "_context_tenant_name": "services",
+ "_context_user": "31055c32b50442e5a4eb4c0f0cb3430b",
+ "_context_user_id": "31055c32b50442e5a4eb4c0f0cb3430b",
+ "payload": {
+ "port": {
+ "status": "DOWN",
+ "binding:host_id": "oscomp-ch2-a06",
+ "name": "",
+ "allowed_address_pairs": [
+
+ ],
+ "admin_state_up": true,
+ "network_id": "af33487a-4e96-4499-bfcd-4f741617a763",
+ "tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "binding:vif_details": {
+ "port_filter": true,
+ "ovs_hybrid_plug": true
+ },
+ "binding:vnic_type": "normal",
+ "binding:vif_type": "ovs",
+ "device_owner": "compute:None",
+ "mac_address": "fa:16:3e:4a:ac:75",
+ "binding:profile": {
+ },
+ "fixed_ips": [
+ {
+ "subnet_id": "4a23cb36-b861-4daa-a8ef-c61360663669",
+ "ip_address": "162.150.0.117"
+ },
+ {
+ "subnet_id": "244c99a6-8011-4177-855b-dd493c5175c5",
+ "ip_address": "2001:558:fe21:403:f816:3eff:fe4a:ac75"
+ }
+ ],
+ "id": "a6c671d7-b4d5-4ebb-afaf-0c822bcc8948",
+ "security_groups": [
+ "0783a151-768c-49d3-a31d-178f70fabd51",
+ "46d46540-98ac-4c93-ae62-68dddab2282e"
+ ],
+ "device_id": "128bc33a-22ae-48b4-8283-093b6ec749d0"
+ }
+ },
+ "_context_project_name": "services",
+ "_context_read_deleted": "no",
+ "_context_tenant": "0b65cf220eab4a3cbd68681d188d7dc7",
+ "priority": "INFO",
+ "_context_is_admin": true,
+ "_context_project_id": "0b65cf220eab4a3cbd68681d188d7dc7",
+ "_context_timestamp": "2016-10-03 18:40:34.477012",
+ "_context_user_name": "neutron",
+ "publisher_id": "network.osctrl-ch2-a03",
+ "message_id": "71047538-531f-4aca-be09-a31bec441d16"
+ }
+
+ `
+
+ securityGroupRuleCreateWithCustomProtocall = `
+ {
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-a17c784c-fec9-4077-8908-44b6f56b6196",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 17:50:59.982008",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"a7452605170c4979b2c6b76911d22026",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule":{
+ "remote_group_id":null,
+ "direction":"ingress",
+ "protocol":10,
+ "remote_ip_prefix":"10.0.0.0/8",
+ "port_range_max":null,
+ "dscp":null,
+ "security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "port_range_min":null,
+ "ethertype":"IPv4",
+ "id":"3eff38bb-eb03-450b-aed4-019d612baeec"
+ }
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 17:50:59.925462",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"6c93e24f-0892-494b-8e68-46252ceb9611"
+ }
+ `
+
+ securityGroupRuleCreateWithIcmpAndCider = `
+ {
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-c584fd21-9e58-4624-b316-b53487eed98e",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 18:05:35.836029",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"cd280fd4f1474266bd0ad6e3ee5933a6",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule":{
+ "remote_group_id":null,
+ "direction":"ingress",
+ "protocol":"icmp",
+ "remote_ip_prefix":"192.168.1.0/24",
+ "port_range_max":null,
+ "dscp":null,
+ "security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "port_range_min":null,
+ "ethertype":"IPv4",
+ "id":"66d7ac79-3551-4436-83c7-103b50760cfb"
+ }
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 18:05:35.769947",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"f67b70d5-a782-4c5e-a274-a7ff197b73ec"
+ }
+ `
+ securityGroupRuleCreateWithports = `
+ {
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-1f17d667-c33f-4fa4-a026-8e2872dbf1d8",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 17:32:25.723344",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"2fad8ecdd86e4748850d91bb0c83d625",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule":{
+ "remote_group_id":null,
+ "direction":"ingress",
+ "protocol":"tcp",
+ "remote_ip_prefix":"10.0.0.0/8",
+ "port_range_max":443,
+ "dscp":null,
+ "security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "port_range_min":443,
+ "ethertype":"IPv4",
+ "id":"2b84d898-67b4-4370-9808-40a3fdb55a64"
+ }
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 17:32:25.665588",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"4df01871-8bdb-4b85-bb34-cbff59ee6034"
+ }
+ `
+ securityGroupRuleCreateWithSecurityGroupAsRemoteIPPrefix = `
+ {
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-9e0360c7-786f-4a5b-84b6-7d2ccd23cbdd",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 17:36:58.780554",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"b38fe8caed514eb2ba910e1ae74c6321",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule":{
+ "remote_group_id":"0783a151-768c-49d3-a31d-178f70fabd51",
+ "direction":"ingress",
+ "protocol":"tcp",
+ "remote_ip_prefix":null,
+ "port_range_max":25,
+ "dscp":null,
+ "security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "port_range_min":20,
+ "ethertype":"IPv6",
+ "id":"7b14b6cd-f966-4b61-aaad-c03d8eacc830"
+ }
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 17:36:58.712962",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"e2d7c089-8194-4523-8f84-ae22db497f60"
+ }
+ `
+ securityGroupRuleCreateWithSSHOpenToTheInternet = `
+ {
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-94df69c6-1c3f-48bd-b2f6-f47abdef5d9b",
+ "event_type":"security_group_rule.create.end",
+ "timestamp":"2016-10-03 18:09:11.938476",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"09412fff881543679f30412ef2342954",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule":{
+ "remote_group_id":null,
+ "direction":"ingress",
+ "protocol":"tcp",
+ "remote_ip_prefix":"0.0.0.0/0",
+ "port_range_max":22,
+ "dscp":null,
+ "security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "port_range_min":22,
+ "ethertype":"IPv4",
+ "id":"bf288dfc-f9cb-446b-bacc-a8de016c9b11"
+ }
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 18:09:11.876789",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"afb043b6-fa56-470b-b17e-984fb4cb6505"
+ }
+ `
+ securityGroupRuleDeleteWithIcmpAndCider = `
+ {
+ "_context_roles": [
+ "Member"
+ ],
+ "_context_request_id": "req-836eb80f-c6eb-459b-87b6-a093ebac3051",
+ "event_type": "security_group_rule.delete.end",
+ "timestamp": "2016-10-03 18:14:33.007074",
+ "_context_tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id": "04beeb34769b43bca09ec837d86ed18b",
+ "_context_tenant_name": "VOIP",
+ "_context_user": "bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "payload": {
+ "security_group_rule_id": "7b14b6cd-f966-4b61-aaad-c03d8eacc830"
+ },
+ "_context_project_name": "VOIP",
+ "_context_read_deleted": "no",
+ "_context_tenant": "ada3b9b0dbac429f9361e803b54f5f32",
+ "priority": "INFO",
+ "_context_is_admin": false,
+ "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp": "2016-10-03 18:14:32.962116",
+ "_context_user_name": "admin",
+ "publisher_id": "network.osctrl-ch2-a03",
+ "message_id": "9bc5106c-a08b-4cda-9311-20bc16bc3008"
+ }
+ `
+)
diff --git a/events_test.go b/events_test.go
new file mode 100644
index 0000000..7dbcb23
--- /dev/null
+++ b/events_test.go
@@ -0,0 +1,61 @@
+package main
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestParseEventWillReturnAnEventStruct(t *testing.T) {
+ event, err := ParseEvent([]byte(securityGroupRuleCreateWithports))
+ assert.Nil(t, err)
+ assert.Equal(t, "main.Event", reflect.TypeOf(event).String(),
+ "ParseEvent should return an Event struct")
+ assert.Equal(t, "security_group_rule.create.end", event.EventData.EventType)
+ assert.Equal(t, "bca89c1b248e4aef9c69ece9e744cc54", event.EventData.UserID)
+ assert.Equal(t, "admin", event.EventData.UserName)
+ assert.Equal(t, "ada3b9b0dbac429f9361e803b54f5f32", event.EventData.TenantID)
+ assert.Equal(t, "VOIP", event.EventData.TenantName)
+}
+
+func TestParseEventWillCreateTheProperEventProcessor(t *testing.T) {
+ e, err := ParseEvent([]byte(securityGroupRuleCreateWithports))
+ assert.Nil(t, err)
+ //assert.Equal(t, "main.EventSecurityGroupRuleChange", reflect.TypeOf(e.Processor).String(),
+ // "ParseEvent should return the proper implementation of EventProcessor")
+ assert.Equal(t, EventSecurityGroupRuleChange{"sg_rule_add"}, e.Processor,
+ "ParseEvent should return the proper implementation of EventProcessor")
+
+ e, err = ParseEvent([]byte(securityGroupRuleDeleteWithIcmpAndCider))
+ assert.Nil(t, err)
+ assert.Equal(t, "main.EventSecurityGroupRuleChange", reflect.TypeOf(e.Processor).String(),
+ "ParseEvent should return the proper implementation of EventProcessor")
+
+ // _, eventProcessor, err = ParseEvent([]byte(portCreateWhenCreatingInstance))
+ // assert.Nil(t, err)
+ // assert.Equal(t, "main.EventPortChange", reflect.TypeOf(eventProcessor).String(),
+ // "ParseEvent should return the proper implementation of EventProcessor")
+
+}
+
+// func TestPortCreateEvent(t *testing.T) {
+// fakeOpenStack := connectFakeOpenstack()
+// event, eventProcessor, err := ParseEvent([]byte(portCreateWhenCreatingInstance))
+// assert.Nil(t, err)
+// eventProcessor.FillExtraData(&event, fakeOpenStack)
+//}
+
+func TestEventSecurityGroupRuleCreateEvent(t *testing.T) {
+ fakeOpenStack := connectFakeOpenstack()
+ event, err := ParseEvent([]byte(securityGroupRuleCreateWithports))
+ assert.Nil(t, err)
+ event.Processor.FillExtraData(&event, fakeOpenStack)
+}
+
+func TestEventSecurityGroupRuleDeleteEvent(t *testing.T) {
+ fakeOpenStack := connectFakeOpenstack()
+ event, err := ParseEvent([]byte(securityGroupRuleDeleteWithIcmpAndCider))
+ assert.Nil(t, err)
+ event.Processor.FillExtraData(&event, fakeOpenStack)
+}
diff --git a/fixtures/README.md b/fixtures/README.md
new file mode 100644
index 0000000..1831460
--- /dev/null
+++ b/fixtures/README.md
@@ -0,0 +1,6 @@
+# Test Fixtures
+
+The fixtures folder is meant to hold test files that can be used to create positive and negative tests.
+
+The test files should be organized in a folder per _test.go file. The folder should be named after the test. For example foo_test.go
+should use a folder called foo. Sub folders are fine. Just don't go nuts and KISS
\ No newline at end of file
diff --git a/fixtures/event_processor/icehouse/ignore_events/compute_instance_create_start.json b/fixtures/event_processor/icehouse/ignore_events/compute_instance_create_start.json
new file mode 100644
index 0000000..909864e
--- /dev/null
+++ b/fixtures/event_processor/icehouse/ignore_events/compute_instance_create_start.json
@@ -0,0 +1,90 @@
+{
+ "_context_roles": [
+ "Member",
+ "admin"
+ ],
+ "_context_request_id": "req-49d3158a-1288-4b76-bed9-907666d7d8c5",
+ "_context_quota_class": null,
+ "event_type": "compute.instance.create.start",
+ "_context_service_catalog": [
+ {
+ "endpoints": [
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "public",
+ "region": "ndc_ch2_a",
+ "id": "7d0a1277410648d4a3e0164ad233adea"
+ },
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "admin",
+ "region": "ndc_ch2_a",
+ "id": "df06789d23684643a0b59bfa7ee26064"
+ }
+ ],
+ "type": "volume",
+ "id": "e90968e859474737a4ad7aadffb2f578"
+ }
+ ],
+ "timestamp": "2016-10-03 18:40:33.699871",
+ "_context_user": "bca89c1b248e4aef9c69ece9e744cc54",
+ "_unique_id": "ab304ee51028479d899f38cd100b9b36",
+ "_context_instance_lock_checked": false,
+ "_context_user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "payload": {
+ "state_description": "scheduling",
+ "availability_zone": null,
+ "terminated_at": "",
+ "ephemeral_gb": 0,
+ "instance_type_id": 2,
+ "message": "",
+ "deleted_at": "",
+ "reservation_id": "r-vgn2gv93",
+ "instance_id": "128bc33a-22ae-48b4-8283-093b6ec749d0",
+ "user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "hostname": "test-sc-group",
+ "state": "building",
+ "launched_at": "",
+ "metadata": {
+ },
+ "node": null,
+ "ramdisk_id": "",
+ "access_ip_v6": null,
+ "disk_gb": 1,
+ "access_ip_v4": null,
+ "kernel_id": "",
+ "image_name": "cirros-0.3.4",
+ "host": null,
+ "display_name": "test_sc_group",
+ "image_ref_url": "http://172.28.195.37:9292/images/caf23aab-f6ec-4691-8f50-b0c70a33eca3",
+ "root_gb": 1,
+ "tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "created_at": "2016-10-03T18:40:33.000000",
+ "memory_mb": 512,
+ "instance_type": "m1.tiny",
+ "vcpus": 1,
+ "image_meta": {
+ "description": "cirros-0.3.4",
+ "container_format": "bare",
+ "min_ram": "0",
+ "disk_format": "qcow2",
+ "min_disk": "1",
+ "base_image_ref": "caf23aab-f6ec-4691-8f50-b0c70a33eca3"
+ },
+ "architecture": null,
+ "os_type": null,
+ "instance_flavor_id": "1"
+ },
+ "_context_project_name": "VOIP",
+ "_context_read_deleted": "no",
+ "_context_auth_token": "7b82d6dec2eb457d85e08940a7f7b33e",
+ "_context_tenant": "ada3b9b0dbac429f9361e803b54f5f32",
+ "priority": "INFO",
+ "_context_is_admin": false,
+ "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp": "2016-10-03T18:40:32.636697",
+ "_context_user_name": "admin",
+ "publisher_id": "compute.oscomp-ch2-a06",
+ "message_id": "0553fb1f-42d1-4959-8991-4b5034701d9d",
+ "_context_remote_address": "127.0.0.1"
+}
diff --git a/fixtures/event_processor/icehouse/ignore_events/compute_instance_delete_end.json b/fixtures/event_processor/icehouse/ignore_events/compute_instance_delete_end.json
new file mode 100644
index 0000000..ca562e2
--- /dev/null
+++ b/fixtures/event_processor/icehouse/ignore_events/compute_instance_delete_end.json
@@ -0,0 +1,88 @@
+{
+ "_context_roles": [
+ "Member",
+ "admin"
+ ],
+ "_context_request_id": "req-dceb796f-f5cf-45b0-8f6c-78b669304f4d",
+ "_context_quota_class": null,
+ "event_type": "compute.instance.delete.end",
+ "_context_service_catalog": [
+ {
+ "endpoints": [
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "public",
+ "region": "ndc_ch2_a",
+ "id": "7d0a1277410648d4a3e0164ad233adea"
+ },
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "admin",
+ "region": "ndc_ch2_a",
+ "id": "df06789d23684643a0b59bfa7ee26064"
+ }
+ ],
+ "type": "volume",
+ "id": "e90968e859474737a4ad7aadffb2f578"
+ }
+ ],
+ "timestamp": "2016-10-03 18:25:18.941599",
+ "_context_user": "bca89c1b248e4aef9c69ece9e744cc54",
+ "_unique_id": "a4c13937ec624259856d11e0ae0717cf",
+ "_context_instance_lock_checked": false,
+ "_context_user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "payload": {
+ "state_description": "",
+ "availability_zone": null,
+ "terminated_at": "2016-10-03T18:25:18.000000",
+ "ephemeral_gb": 0,
+ "instance_type_id": 2,
+ "deleted_at": "2016-10-03T18:25:18.857623",
+ "reservation_id": "r-osn0qo9l",
+ "instance_id": "563d0899-d1cc-4154-bb58-f932ad0b255c",
+ "user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "hostname": "test",
+ "state": "deleted",
+ "launched_at": "2016-10-03T18:23:11.000000",
+ "metadata": {
+ },
+ "node": "openstack-host2",
+ "ramdisk_id": "",
+ "access_ip_v6": null,
+ "disk_gb": 1,
+ "access_ip_v4": null,
+ "kernel_id": "",
+ "host": "oscomp-ch2-a07",
+ "display_name": "test",
+ "image_ref_url": "http://172.28.195.38:9292/images/caf23aab-f6ec-4691-8f50-b0c70a33eca3",
+ "root_gb": 1,
+ "tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "created_at": "2016-10-03 18:23:03+00:00",
+ "memory_mb": 512,
+ "instance_type": "m1.tiny",
+ "vcpus": 1,
+ "image_meta": {
+ "description": "cirros-0.3.4",
+ "container_format": "bare",
+ "min_ram": "0",
+ "disk_format": "qcow2",
+ "min_disk": "1",
+ "base_image_ref": "caf23aab-f6ec-4691-8f50-b0c70a33eca3"
+ },
+ "architecture": null,
+ "os_type": null,
+ "instance_flavor_id": "1"
+ },
+ "_context_project_name": "VOIP",
+ "_context_read_deleted": "no",
+ "_context_auth_token": "7b82d6dec2eb457d85e08940a7f7b33e",
+ "_context_tenant": "ada3b9b0dbac429f9361e803b54f5f32",
+ "priority": "INFO",
+ "_context_is_admin": false,
+ "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp": "2016-10-03T18:25:16.888382",
+ "_context_user_name": "admin",
+ "publisher_id": "compute.oscomp-ch2-a07",
+ "message_id": "cf848ea3-300e-4b9a-8672-2306e9413cbf",
+ "_context_remote_address": "127.0.0.1"
+}
diff --git a/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_end.json b/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_end.json
new file mode 100644
index 0000000..159668c
--- /dev/null
+++ b/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_end.json
@@ -0,0 +1,88 @@
+{
+ "_context_roles": [
+ "Member",
+ "admin"
+ ],
+ "_context_request_id": "req-71eec143-a576-49aa-84f6-eeaccd4f6619",
+ "_context_quota_class": null,
+ "event_type": "compute.instance.shutdown.end",
+ "_context_service_catalog": [
+ {
+ "endpoints": [
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "public",
+ "region": "ndc_ch2_a",
+ "id": "7d0a1277410648d4a3e0164ad233adea"
+ },
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "admin",
+ "region": "ndc_ch2_a",
+ "id": "df06789d23684643a0b59bfa7ee26064"
+ }
+ ],
+ "type": "volume",
+ "id": "e90968e859474737a4ad7aadffb2f578"
+ }
+ ],
+ "timestamp": "2016-10-03 18:38:47.143591",
+ "_context_user": "bca89c1b248e4aef9c69ece9e744cc54",
+ "_unique_id": "2740b7baa2fb48fea3c3654bf94c304e",
+ "_context_instance_lock_checked": false,
+ "_context_user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "payload": {
+ "state_description": "deleting",
+ "availability_zone": null,
+ "terminated_at": "",
+ "ephemeral_gb": 0,
+ "instance_type_id": 2,
+ "deleted_at": "",
+ "reservation_id": "r-s8qu0d07",
+ "instance_id": "8bca63d6-0d51-465b-9379-c20ccc8d3e17",
+ "user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "hostname": "test-instance-one",
+ "state": "active",
+ "launched_at": "2016-10-03T17:23:59.000000",
+ "metadata": {
+ },
+ "node": "openstack-host1",
+ "ramdisk_id": "",
+ "access_ip_v6": null,
+ "disk_gb": 1,
+ "access_ip_v4": null,
+ "kernel_id": "",
+ "host": "oscomp-ch2-a06",
+ "display_name": "test_instance_one",
+ "image_ref_url": "http://172.28.195.37:9292/images/caf23aab-f6ec-4691-8f50-b0c70a33eca3",
+ "root_gb": 1,
+ "tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "created_at": "2016-10-03 17:23:40+00:00",
+ "memory_mb": 512,
+ "instance_type": "m1.tiny",
+ "vcpus": 1,
+ "image_meta": {
+ "description": "cirros-0.3.4",
+ "container_format": "bare",
+ "min_ram": "0",
+ "disk_format": "qcow2",
+ "min_disk": "1",
+ "base_image_ref": "caf23aab-f6ec-4691-8f50-b0c70a33eca3"
+ },
+ "architecture": null,
+ "os_type": null,
+ "instance_flavor_id": "1"
+ },
+ "_context_project_name": "VOIP",
+ "_context_read_deleted": "no",
+ "_context_auth_token": "7b82d6dec2eb457d85e08940a7f7b33e",
+ "_context_tenant": "ada3b9b0dbac429f9361e803b54f5f32",
+ "priority": "INFO",
+ "_context_is_admin": true,
+ "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp": "2016-10-03T18:38:45.381205",
+ "_context_user_name": "admin",
+ "publisher_id": "compute.oscomp-ch2-a06",
+ "message_id": "1cffe568-1eee-4150-89a0-937973ce15e4",
+ "_context_remote_address": "127.0.0.1"
+}
diff --git a/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_start.json b/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_start.json
new file mode 100644
index 0000000..0b4fe14
--- /dev/null
+++ b/fixtures/event_processor/icehouse/ignore_events/compute_instance_shutdown_start.json
@@ -0,0 +1,88 @@
+{
+ "_context_roles": [
+ "Member",
+ "admin"
+ ],
+ "_context_request_id": "req-dceb796f-f5cf-45b0-8f6c-78b669304f4d",
+ "_context_quota_class": null,
+ "event_type": "compute.instance.shutdown.start",
+ "_context_service_catalog": [
+ {
+ "endpoints": [
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "public",
+ "region": "ndc_ch2_a",
+ "id": "7d0a1277410648d4a3e0164ad233adea"
+ },
+ {
+ "url": "https://openstack-host:8776/v1/ada3b9b0dbac429f9361e803b54f5f32",
+ "interface": "admin",
+ "region": "ndc_ch2_a",
+ "id": "df06789d23684643a0b59bfa7ee26064"
+ }
+ ],
+ "type": "volume",
+ "id": "e90968e859474737a4ad7aadffb2f578"
+ }
+ ],
+ "timestamp": "2016-10-03 18:25:17.121727",
+ "_context_user": "bca89c1b248e4aef9c69ece9e744cc54",
+ "_unique_id": "b7c7128bd595423daeed9d6ad36c2b2a",
+ "_context_instance_lock_checked": false,
+ "_context_user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "payload": {
+ "state_description": "deleting",
+ "availability_zone": null,
+ "terminated_at": "",
+ "ephemeral_gb": 0,
+ "instance_type_id": 2,
+ "deleted_at": "",
+ "reservation_id": "r-osn0qo9l",
+ "instance_id": "563d0899-d1cc-4154-bb58-f932ad0b255c",
+ "user_id": "bca89c1b248e4aef9c69ece9e744cc54",
+ "hostname": "test",
+ "state": "active",
+ "launched_at": "2016-10-03T18:23:11.000000",
+ "metadata": {
+ },
+ "node": "openstack-host2",
+ "ramdisk_id": "",
+ "access_ip_v6": null,
+ "disk_gb": 1,
+ "access_ip_v4": null,
+ "kernel_id": "",
+ "host": "oscomp-ch2-a07",
+ "display_name": "test",
+ "image_ref_url": "http://172.28.195.38:9292/images/caf23aab-f6ec-4691-8f50-b0c70a33eca3",
+ "root_gb": 1,
+ "tenant_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "created_at": "2016-10-03 18:23:03+00:00",
+ "memory_mb": 512,
+ "instance_type": "m1.tiny",
+ "vcpus": 1,
+ "image_meta": {
+ "description": "cirros-0.3.4",
+ "container_format": "bare",
+ "min_ram": "0",
+ "disk_format": "qcow2",
+ "min_disk": "1",
+ "base_image_ref": "caf23aab-f6ec-4691-8f50-b0c70a33eca3"
+ },
+ "architecture": null,
+ "os_type": null,
+ "instance_flavor_id": "1"
+ },
+ "_context_project_name": "VOIP",
+ "_context_read_deleted": "no",
+ "_context_auth_token": "7b82d6dec2eb457d85e08940a7f7b33e",
+ "_context_tenant": "ada3b9b0dbac429f9361e803b54f5f32",
+ "priority": "INFO",
+ "_context_is_admin": true,
+ "_context_project_id": "ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp": "2016-10-03T18:25:16.888382",
+ "_context_user_name": "admin",
+ "publisher_id": "compute.oscomp-ch2-a07",
+ "message_id": "41963fa2-be81-42ee-b31e-e2f727f109b7",
+ "_context_remote_address": "127.0.0.1"
+}
diff --git a/fixtures/event_processor/icehouse/port_create_with_default_security_group.json b/fixtures/event_processor/icehouse/port_create_with_default_security_group.json
new file mode 100644
index 0000000..289167e
--- /dev/null
+++ b/fixtures/event_processor/icehouse/port_create_with_default_security_group.json
@@ -0,0 +1,36 @@
+{
+ "_context_roles":[
+ "admin"
+ ],
+ "_context_request_id":"req-b22260da-ecd2-4eb6-a7bb-e07a216450de",
+ "event_type":"port.create.start",
+ "timestamp":"2016-10-03 18:23:04.894439",
+ "_context_tenant_id":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "_unique_id":"d133f63bb45740f587b780d4dac7e2ee",
+ "_context_tenant_name":"services",
+ "_context_user":"31055c32b50442e5a4eb4c0f0cb3430b",
+ "_context_user_id":"31055c32b50442e5a4eb4c0f0cb3430b",
+ "payload":{
+ "port":{
+ "binding:host_id":"oscomp-ch2-a07",
+ "admin_state_up":true,
+ "network_id":"af33487a-4e96-4499-bfcd-4f741617a763",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "device_owner":"compute:None",
+ "security_groups":[
+ "0783a151-768c-49d3-a31d-178f70fabd51"
+ ],
+ "device_id":"563d0899-d1cc-4154-bb58-f932ad0b255c"
+ }
+ },
+ "_context_project_name":"services",
+ "_context_read_deleted":"no",
+ "_context_tenant":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "priority":"INFO",
+ "_context_is_admin":true,
+ "_context_project_id":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "_context_timestamp":"2016-10-03 18:23:04.892974",
+ "_context_user_name":"neutron",
+ "publisher_id":"network.osctrl-ch2-a02",
+ "message_id":"b256daa5-3eed-4fec-8c54-d93fa79ab753"
+}
\ No newline at end of file
diff --git a/fixtures/event_processor/icehouse/port_create_with_security_group_attached.json b/fixtures/event_processor/icehouse/port_create_with_security_group_attached.json
new file mode 100644
index 0000000..6d014b8
--- /dev/null
+++ b/fixtures/event_processor/icehouse/port_create_with_security_group_attached.json
@@ -0,0 +1,38 @@
+{
+ "_context_roles":[
+ "admin"
+ ],
+ "_context_request_id":"req-662e5998-9dcd-4b76-a726-5d815a1ab9ef",
+ "event_type":"port.create.start",
+ "timestamp":"2016-10-03 17:25:04.189288",
+ "_context_tenant_id":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "_unique_id":"edd46be30b524fe9a3af78cec809db0e",
+ "_context_tenant_name":"services",
+ "_context_user":"31055c32b50442e5a4eb4c0f0cb3430b",
+ "_context_user_id":"31055c32b50442e5a4eb4c0f0cb3430b",
+ "payload":{
+ "port":{
+ "binding:host_id":"oscomp-ch2-a07",
+ "admin_state_up":true,
+ "network_id":"af33487a-4e96-4499-bfcd-4f741617a763",
+ "tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "device_owner":"compute:None",
+ "security_groups":[
+ "0783a151-768c-49d3-a31d-178f70fabd51",
+ "46d46540-98ac-4c93-ae62-68dddab2282e"
+ ],
+ "device_id":"bb64fe08-0eae-4c83-8bc2-457b6cb7e9a3"
+ }
+ },
+ "_context_project_name":"services",
+ "_context_read_deleted":"no",
+ "_context_tenant":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "priority":"INFO",
+ "_context_is_admin":true,
+ "_context_project_id":"0b65cf220eab4a3cbd68681d188d7dc7",
+ "_context_timestamp":"2016-10-03 17:25:04.187847",
+ "_context_user_name":"neutron",
+ "publisher_id":"network.osctrl-ch2-a02",
+ "message_id":"56253f06-7350-47c0-8284-d74701d11698"
+}
+
diff --git a/fixtures/event_processor/icehouse/port_delete_with_default_security_group.json b/fixtures/event_processor/icehouse/port_delete_with_default_security_group.json
new file mode 100644
index 0000000..be22f4d
--- /dev/null
+++ b/fixtures/event_processor/icehouse/port_delete_with_default_security_group.json
@@ -0,0 +1,26 @@
+{
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-61fa3b99-0582-43a3-a5da-71465e5b56b6",
+ "event_type":"port.delete.end",
+ "timestamp":"2016-10-03 18:25:18.645306",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"7296f75a9e0f4993bde5cc2b6d5cbe7d",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "port_id":"d781dc1e-f940-4de8-ad2f-b1286b75efe0"
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 18:25:18.562793",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a02",
+ "message_id":"3375a032-8bd2-45b1-b3d8-29bdc5aacbf5"
+}
\ No newline at end of file
diff --git a/fixtures/event_processor/icehouse/port_delete_with_security_group_attached.json b/fixtures/event_processor/icehouse/port_delete_with_security_group_attached.json
new file mode 100644
index 0000000..aa53d30
--- /dev/null
+++ b/fixtures/event_processor/icehouse/port_delete_with_security_group_attached.json
@@ -0,0 +1,28 @@
+{
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-d9daa316-5075-4702-961a-6a808db40c1e",
+ "event_type":"port.delete.end",
+ "timestamp":"2016-10-03 17:28:04.004860",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"e7a124d2056c476696a5efada8ddfbf2",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "port_id":"57545a0d-ab4e-4a18-bba2-d7ee8bc9c3c1"
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 17:28:03.916009",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a02",
+ "message_id":"9573b2e3-9f33-499d-ad31-43840f5c1c58"
+}
+
+
diff --git a/fixtures/event_processor/icehouse/security_group_rule_delete_with_ssh_open_to_the_internet.json b/fixtures/event_processor/icehouse/security_group_rule_delete_with_ssh_open_to_the_internet.json
new file mode 100644
index 0000000..d0b2325
--- /dev/null
+++ b/fixtures/event_processor/icehouse/security_group_rule_delete_with_ssh_open_to_the_internet.json
@@ -0,0 +1,26 @@
+{
+ "_context_roles":[
+ "Member"
+ ],
+ "_context_request_id":"req-f96ea9a5-435e-4177-8e51-ebb60d0fae2a",
+ "event_type":"security_group_rule.delete.end",
+ "timestamp":"2016-10-03 18:10:59.112712",
+ "_context_tenant_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_unique_id":"eafc9362327442b49d8c03b0e88d0216",
+ "_context_tenant_name":"VOIP",
+ "_context_user":"bca89c1b248e4aef9c69ece9e744cc54",
+ "_context_user_id":"bca89c1b248e4aef9c69ece9e744cc54",
+ "payload":{
+ "security_group_rule_id":"bf288dfc-f9cb-446b-bacc-a8de016c9b11"
+ },
+ "_context_project_name":"VOIP",
+ "_context_read_deleted":"no",
+ "_context_tenant":"ada3b9b0dbac429f9361e803b54f5f32",
+ "priority":"INFO",
+ "_context_is_admin":false,
+ "_context_project_id":"ada3b9b0dbac429f9361e803b54f5f32",
+ "_context_timestamp":"2016-10-03 18:10:59.079179",
+ "_context_user_name":"admin",
+ "publisher_id":"network.osctrl-ch2-a03",
+ "message_id":"e75fb2ee-85bf-44ba-a083-2445eca2ae10"
+}
\ No newline at end of file
diff --git a/fixtures/viper/test.yml b/fixtures/viper/test.yml
new file mode 100644
index 0000000..e99b2d3
--- /dev/null
+++ b/fixtures/viper/test.yml
@@ -0,0 +1,6 @@
+required_string: "required_string value"
+test_alias: "test_alias value"
+nested:
+ one: "nested.one value"
+
+
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..67b9967
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,20 @@
+module "git.openstack.org/openstack/osel"
+
+require (
+ "github.com/fsnotify/fsnotify" v1.4.7
+ "github.com/google/go-querystring" v0.0.0-20170111101155-53e6ce116135
+ "github.com/hashicorp/hcl" v0.0.0-20171017181929-23c074d0eceb
+ "github.com/magiconair/properties" v1.7.6
+ "github.com/mitchellh/mapstructure" v0.0.0-20180220230111-00c29f56e238
+ "github.com/nate-johnston/viper" v1.0.1
+ "github.com/pelletier/go-toml" v1.1.0
+ "github.com/racker/perigee" v0.1.0
+ "github.com/rackspace/gophercloud" v1.0.0
+ "github.com/spf13/afero" v1.0.2
+ "github.com/spf13/cast" v1.2.0
+ "github.com/spf13/pflag" v1.0.0
+ "github.com/streadway/amqp" v0.0.0-20180131094250-fc7fda2371f5
+ "golang.org/x/sys" v0.0.0-20180302081741-dd2ff4accc09
+ "golang.org/x/text" v0.0.0-20171214130843-f21a4dfb5e38
+ "gopkg.in/yaml.v2" v1.1.1-gopkgin-v2.1.1
+)
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..c2efcc9
--- /dev/null
+++ b/main.go
@@ -0,0 +1,129 @@
+package main // import "git.openstack.org/openstack/osel"
+
+import (
+ "log"
+ "net/url"
+ "os"
+ "time"
+
+ "github.com/fsnotify/fsnotify"
+ "github.com/nate-johnston/viper"
+)
+
+// OselVersion is exposed in the logged JSON in the "source_type" field. This
+// will allow us to track the version of the logging specification.
+// 1.0: Initial revision
+// 1.1: Added qualys_scan_id and qualys_scan_error
+const OselVersion = "osel1.1"
+
+// Debug is a global variable to toggle debug logging
+var Debug bool
+
+// RabbitMQ URI
+var rabbitURI string
+
+func main() {
+ // Declare the configuration
+ viperConfigs := []ViperConfig{
+ ViperConfig{Key: "batch_interval", Description: "Interval of time in minutes for message batching"},
+ ViperConfig{Key: "debug", Description: "Output additional messages for debugging"},
+ ViperConfig{Key: "rabbit_uri", Description: "AMQP connection uri. See: https://www.rabbitmq.com/uri-spec.html"},
+ ViperConfig{Key: "openstack.identity_endpoint", Description: "Openstack Keystone Endpoint"},
+ ViperConfig{Key: "openstack.user", Description: "Openstack user that has at least read only access to all tenants/ports/security groups in the region."},
+ ViperConfig{Key: "openstack.password", Description: "Password for the Openstack user"},
+ ViperConfig{Key: "openstack.region", Description: "The name of the region running this process"},
+ ViperConfig{Key: "qualys.drop6", Description: "Should IPv6 addresses be incorporated in Qualys scans? true or false."},
+ ViperConfig{Key: "qualys.username", Description: "Username for credentials for the Qualys external scanning service"},
+ ViperConfig{Key: "qualys.password", Description: "Password for credentials for the Qualys external scanning service"},
+ ViperConfig{Key: "qualys.url", Description: "URL for thw Qualys service"},
+ ViperConfig{Key: "qualys.proxy_url", Description: "URL for an HTTP proxy that will permit access to the Qualys service"},
+ ViperConfig{Key: "syslog_server", Description: "FQDN of the server for events to log to over the network"},
+ ViperConfig{Key: "syslog_port", Description: "Port for communication to syslog, defaults to 514"},
+ ViperConfig{Key: "syslog_protocol", Description: "tcp or udp, defaults to tcp"},
+ ViperConfig{Key: "retry_syslog", Description: "Should the process keep trying if it cannot reach syslog? true or false."},
+ }
+ configPath := os.Getenv("EL_CONFIG") //The config path comes from ENV.
+ if configPath == "" {
+ log.Fatalln("Fatal Error: The Config file was not set to EL_CONFIG.")
+ }
+ if err := InitViper(configPath, viperConfigs); err != nil {
+ log.Fatalf("Fatal Error: (%s) while reading config file %s", err, configPath)
+ }
+
+ // Set defaults
+ viper.SetDefault("batch_interval", 60)
+ viper.SetDefault("debug", true)
+ viper.SetDefault("qualys.drop6", true)
+ viper.SetDefault("qualys.url", "https://qualysapi.qualys.com/api/2.0/fo/scan/")
+ viper.SetDefault("syslog_port", "514")
+ viper.SetDefault("syslog_protocol", "tcp")
+
+ // Watch for config changes
+ viper.WatchConfig()
+ viper.OnConfigChange(func(fsnotify.Event) {
+ if err := ValidateConfig(viperConfigs); err != nil {
+ log.Printf("Fatal Error: %s while refreshing config file %s\n", err, configPath)
+ }
+ })
+
+ batchInterval := viper.GetInt("batch_interval")
+ Debug = viper.GetBool("debug")
+
+ // Initialize AMQP
+ rabbitURI = viper.GetString("rabbit_uri")
+ amqpBus := new(AmqpActions)
+ amqpBus.Options = AmqpOptions{
+ RabbitURI: rabbitURI,
+ }
+ amqpIncoming, amqpErrorNotify, err := amqpBus.Connect()
+ if err != nil {
+ log.Fatalln(err)
+ }
+
+ // Initialize Qualys
+ qualysURL, err := url.Parse(viper.GetString("qualys.url"))
+ if err != nil {
+ log.Fatal(err)
+ }
+ qualysProxyURL, err := url.Parse(viper.GetString("qualys.proxy_url"))
+ if err != nil {
+ log.Fatal(err)
+ }
+ qualys := new(QualysActions)
+ qualys.Options = QualysOptions{
+ DropIPv6: viper.GetBool("qualys.drop6"),
+ Password: viper.GetString("qualys.password"),
+ ProxyURL: qualysProxyURL,
+ QualysURL: qualysURL,
+ ScanOptionName: viper.GetString("qualys.option"),
+ MinRemaining: viper.GetInt("qualys.min_remaining"),
+ UserName: viper.GetString("qualys.username"),
+ }
+
+ // Initialize OpenStack
+ openstack := new(OpenStackActions)
+ openstack.Options = OpenStackOptions{
+ KeystoneURI: viper.GetString("openstack.identity_endpoint"),
+ Password: viper.GetString("openstack.password"),
+ RegionName: viper.GetString("openstack.region"),
+ UserName: viper.GetString("openstack.user"),
+ }
+
+ // Initialize Syslog
+ logger := new(SyslogActions)
+ logger.Options = SyslogOptions{
+ Host: viper.GetString("syslog_server"),
+ Port: viper.GetString("syslog_port"),
+ Protocol: viper.GetString("syslog_protocol"),
+ Retry: viper.GetBool("retry_syslog"),
+ }
+ err = logger.Connect()
+ if err != nil {
+ log.Fatal(err)
+ }
+
+ // run main loop
+ batchDuration := time.Duration(batchInterval) * time.Minute
+ mainLoop(batchDuration, amqpIncoming, amqpErrorNotify, openstack, logger, qualys)
+ defer amqpBus.Close()
+}
diff --git a/openstack.go b/openstack.go
new file mode 100644
index 0000000..ebe96a6
--- /dev/null
+++ b/openstack.go
@@ -0,0 +1,126 @@
+package main
+
+/*
+
+openstack - This file includes all of the logic necessary to interact with
+OpenStack. This is extrapolated out so that an OpenStackActioner
+interface can be passed to functions. Doing this allows testing by mock
+classes to be created that can be passed to functions.
+
+Since this is a wrapper around the gophercloud libraries, this does not need
+testing.
+
+*/
+
+import (
+ "fmt"
+ "log"
+
+ "github.com/rackspace/gophercloud"
+ "github.com/rackspace/gophercloud/openstack"
+ "github.com/rackspace/gophercloud/openstack/networking/v2/ports"
+ "github.com/rackspace/gophercloud/pagination"
+)
+
+// OpenStackActioner is an interface for an OpenStackActions class.
+// Having this as an interface allows us to pass in a dummy class for testing
+// that just returns mocked data.
+type OpenStackActioner interface {
+ GetPortList() []OpenStackIPMap
+ Connect(string, string) error
+}
+
+// OpenStackActions is a class that handles all interactions directly with
+// OpenStack. See the comment on OpenStackActioner for rationale.
+type OpenStackActions struct {
+ gopherCloudClient *gophercloud.ProviderClient
+ neutronClient *gophercloud.ServiceClient
+ Options OpenStackOptions
+}
+
+// OpenStackOptions is a class to convey all of the configurable options for the
+// OpenStackActions class.
+type OpenStackOptions struct {
+ KeystoneURI string
+ Password string
+ RegionName string
+ TenantID string
+ UserName string
+}
+
+// OpenStackIPMap is a struct that is used to capture the mapping of IP address
+// to security group. It is what is returned, in array form, from port list.
+type OpenStackIPMap struct {
+ ipAddress string
+ securityGroup string
+}
+
+// GetPortList is a method that uses GopherCloud to query OpenStack for a
+// list of ports, with their associated security group. It returns an array of
+// OpenStackIPMap.
+func (s OpenStackActions) GetPortList() []OpenStackIPMap {
+ // Make port list request to neutron
+ var ips []OpenStackIPMap
+ portListOpts := ports.ListOpts{
+ TenantID: s.Options.TenantID,
+ }
+ if s.neutronClient == nil {
+ log.Println("Error: neutronClient is nil")
+ }
+ pager := ports.List(s.neutronClient, portListOpts)
+
+ // Define an anonymous function to be executed on each page's iteration
+ pager.EachPage(func(page pagination.Page) (bool, error) {
+ portList, err := ports.ExtractPorts(page)
+ if err != nil {
+ // ignore ?
+ }
+
+ for _, p := range portList {
+ // "p" will be a ports.Port
+ for _, fixedIP := range p.FixedIPs {
+ for _, securityGroup := range p.SecurityGroups {
+ ips = append(ips, OpenStackIPMap{
+ ipAddress: fixedIP.IPAddress,
+ securityGroup: securityGroup,
+ })
+ }
+ }
+ }
+ return true, err
+ })
+ return ips
+}
+
+// Connect is the method that establishes a connection to the OpenStack
+// service.
+func (s *OpenStackActions) Connect(tenantID string, username string) error {
+ var err error
+ keystoneOpts := gophercloud.AuthOptions{
+ IdentityEndpoint: s.Options.KeystoneURI,
+ TenantID: tenantID,
+ Username: username,
+ Password: s.Options.Password,
+ AllowReauth: true,
+ }
+
+ log.Println(fmt.Sprintf("Connecting to keystone %q in region %q for tenant %q with user %q", s.Options.KeystoneURI,
+ s.Options.RegionName, tenantID, username))
+ s.gopherCloudClient, err = openstack.AuthenticatedClient(keystoneOpts)
+ if err != nil {
+ return fmt.Errorf("unable to connect to %s using user %s for tenant %s: %s",
+ s.Options.KeystoneURI, s.Options.UserName, s.Options.TenantID, err)
+ }
+ log.Println("Connected to gophercloud ", s.Options.KeystoneURI)
+
+ neutronOpts := gophercloud.EndpointOpts{
+ Name: "neutron",
+ Region: s.Options.RegionName,
+ }
+ s.neutronClient, err = openstack.NewNetworkV2(s.gopherCloudClient, neutronOpts)
+ if err != nil {
+ return fmt.Errorf("unable to connect to neutron using user %s in region %s: %s",
+ s.Options.UserName, s.Options.RegionName, err)
+ }
+ return nil
+}
diff --git a/openstack_mock_test.go b/openstack_mock_test.go
new file mode 100644
index 0000000..e593af8
--- /dev/null
+++ b/openstack_mock_test.go
@@ -0,0 +1,31 @@
+package main
+
+type OpenStackTestActions struct {
+ regionName string
+ tenantID string
+}
+
+func (s OpenStackTestActions) Connect(tenantID string, username string) error {
+ return nil
+}
+
+func (s OpenStackTestActions) GetPortList() []OpenStackIPMap {
+ return []OpenStackIPMap{
+ {
+ ipAddress: "10.0.0.1",
+ securityGroup: "46d46540-98ac-4c93-ae62-68dddab2282e",
+ },
+ {
+ ipAddress: "10.0.0.2",
+ securityGroup: "groupTwo",
+ },
+ {
+ ipAddress: "10.0.0.3",
+ securityGroup: "46d46540-98ac-4c93-ae62-68dddab2282e",
+ },
+ }
+}
+
+func connectFakeOpenstack() *OpenStackTestActions {
+ return new(OpenStackTestActions)
+}
diff --git a/processing.go b/processing.go
new file mode 100644
index 0000000..6f630fa
--- /dev/null
+++ b/processing.go
@@ -0,0 +1,133 @@
+package main
+
+import (
+ "fmt"
+ "log"
+ "net"
+ "time"
+
+ "github.com/streadway/amqp"
+)
+
+func processWaitingEvent(delivery amqp.Delivery, openstackActions OpenStackActioner) (Event, error) {
+ // executes when an event is waiting
+ event, err := ParseEvent(delivery.Body)
+ if err != nil {
+ return Event{}, fmt.Errorf("Failed to parse event due to error: %s", err)
+ }
+ if event.Processor == nil {
+ if !Debug {
+ return Event{}, nil
+ }
+ return Event{}, fmt.Errorf("Ignoring event type %s", event.EventData.EventType)
+ }
+ if Debug {
+ log.Printf("Processing event type %s\n", event.EventData.EventType)
+ }
+ err = event.Processor.FillExtraData(&event, openstackActions)
+ if err != nil {
+ return Event{}, fmt.Errorf("Error fetching extra data: %s", err)
+ }
+ return event, nil
+}
+
+func logEvents(events []Event, logger SyslogActioner, qualys QualysActioner) {
+ var ipAddresses []string
+ var qualysIPAddresses []string
+
+ if Debug {
+ log.Println("Timer Expired")
+ }
+
+ // De-dupe IP addresses and get them into a single struct
+ dedupIPAddresses := make(map[string]struct{})
+ for _, event := range events {
+ for _, IPs := range event.IPs {
+ for _, IP := range IPs {
+ if _, ok := dedupIPAddresses[IP]; !ok {
+ ipAddresses = append(ipAddresses, IP)
+ }
+ dedupIPAddresses[IP] = struct{}{}
+ }
+ }
+ }
+
+ // Disregard the scan if no targets have been found
+ if len(ipAddresses) == 0 {
+ if Debug {
+ log.Println("Nothing to scan, skipping...")
+ }
+ return
+ }
+
+ // Remove IPv6 addresses
+ if qualys.DropIPv6() {
+ for ipAddressIndex := range ipAddresses {
+ testIPAddress := ipAddresses[ipAddressIndex]
+ if net.ParseIP(testIPAddress).To4() != nil {
+ qualysIPAddresses = append(qualysIPAddresses, testIPAddress)
+ } else {
+ log.Println("Disregarded IPv6 address", testIPAddress)
+ }
+ }
+ }
+
+ // Execute Qualys scan
+
+ log.Println("Qualys Scan Starting")
+ scanID, scanError := qualys.InitiateScan(qualysIPAddresses)
+ log.Printf("Qualys Scan Complete: scan ID='%s'; scan_error='%v'", scanID, scanError)
+
+ // Iterate through entries and format the logs
+ log.Printf("Processing %d events\n", len(events))
+ for _, event := range events {
+ event.QualysScanID = scanID
+ if scanError != nil {
+ event.QualysScanError = scanError.Error()
+ }
+ event.LogLines, _ = event.Processor.FormatLogs(&event, qualysIPAddresses)
+
+ // Output the logs
+ log.Printf("Processing %d loglines\n", len(event.LogLines))
+ for lineToLog := range event.LogLines {
+ logger.Info(event.LogLines[lineToLog])
+ }
+ }
+}
+
+func mainLoop(batchInterval time.Duration, deliveries <-chan amqp.Delivery, amqpNotifyError chan *amqp.Error, openstackActions OpenStackActioner, logger SyslogActioner, qualys QualysActioner) {
+ var events []Event
+ ticker := time.NewTicker(batchInterval)
+ amqpReconnectTimer := time.NewTimer(1)
+ for {
+ select {
+ case e := <-deliveries:
+ event, err := processWaitingEvent(e, openstackActions)
+ if err != nil {
+ log.Printf("Event skipped: %s\n", err)
+ continue
+ }
+ events = append(events, event)
+ case <-ticker.C:
+ logEvents(events, logger, qualys)
+ events = nil
+ case err := <-amqpNotifyError:
+ // Reinitialize AMQP on connection error
+ log.Printf("AMQP connection error: %s\n", err)
+ amqpReconnectTimer = time.NewTimer(time.Second * 30)
+ case <-amqpReconnectTimer.C:
+ var err error
+ amqpBus := new(AmqpActions)
+ amqpBus.Options = AmqpOptions{
+ RabbitURI: rabbitURI,
+ }
+ deliveries, amqpNotifyError, err = amqpBus.Connect()
+ if err != nil {
+ log.Printf("AMQP retry connection error: %s\n", err)
+ amqpReconnectTimer = time.NewTimer(time.Second * 30)
+ } else {
+ log.Printf("AMQP reconnected\n")
+ }
+ }
+ }
+}
diff --git a/processing_test.go b/processing_test.go
new file mode 100644
index 0000000..e5566b5
--- /dev/null
+++ b/processing_test.go
@@ -0,0 +1,48 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "testing"
+
+ "github.com/streadway/amqp"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestProcessWaitingEvent(t *testing.T) {
+ var delivery amqp.Delivery
+ openstackActions := connectFakeOpenstack()
+
+ delivery.Body = []byte(securityGroupRuleCreateWithIcmpAndCider)
+ event, err := processWaitingEvent(delivery, openstackActions)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ _ = event
+}
+
+func TestLogEvents(t *testing.T) {
+ hostName, _ := os.Hostname()
+ IPList := []string{"10.0.0.1", "10.0.0.3"}
+ logLines := []string{fmt.Sprintf(`{"security_group_rule":{"remote_group_id":null,"direction":"ingress","protocol":"icmp","remote_ip_prefix":"192.168.1.0/24","port_range_max":null,"rule_direction":"","security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e","tenant_id":"ada3b9b0dbac429f9361e803b54f5f32","port_range_min":null,"ethertype":"IPv4","id":"66d7ac79-3551-4436-83c7-103b50760cfb"},"affected_ip_address":"10.0.0.1","change_type":"sg_rule_add","source_type":"osel","source_message_bus":"%s"}`, hostName), fmt.Sprintf(`{"security_group_rule":{"remote_group_id":null,"direction":"ingress","protocol":"icmp","remote_ip_prefix":"192.168.1.0/24","port_range_max":null,"rule_direction":"","security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e","tenant_id":"ada3b9b0dbac429f9361e803b54f5f32","port_range_min":null,"ethertype":"IPv4","id":"66d7ac79-3551-4436-83c7-103b50760cfb"},"affected_ip_address":"10.0.0.3","change_type":"sg_rule_add","source_type":"osel","source_message_bus":"%s"}`, hostName)}
+ logger := connectFakeSyslog()
+ qualys := connectFakeQualys()
+ IPs := make(map[string][]string)
+
+ IPs["46d46540-98ac-4c93-ae62-68dddab2282e"] = IPList
+ fakeEvent := Event{
+ RawData: []byte(securityGroupRuleCreateWithIcmpAndCider),
+ LogLines: logLines,
+ Processor: EventSecurityGroupRuleChange{ChangeType: "sg_rule_add"},
+ IPs: IPs,
+ }
+ events := []Event{fakeEvent}
+
+ logEvents(events, logger, qualys)
+ savedLogs := logger.GetLogs()
+ assert.Equal(t, 2, len(savedLogs))
+
+ logLine1 := fmt.Sprintf(`{"affected_ip_address":"10.0.0.1","change_type":"sg_rule_add","qualys_scan_id":"","qualys_scan_error":"Not scanned by Qualys","security_group_rule":{"remote_group_id":null,"direction":"ingress","protocol":"icmp","remote_ip_prefix":"192.168.1.0/24","port_range_max":null,"rule_direction":"","security_group_id":"46d46540-98ac-4c93-ae62-68dddab2282e","tenant_id":"ada3b9b0dbac429f9361e803b54f5f32","port_range_min":null,"ethertype":"IPv4","id":"66d7ac79-3551-4436-83c7-103b50760cfb"},"source_type":"osel1.1","source_message_bus":"%s"}`, hostName)
+ assert.Equal(t, logLine1, savedLogs[0])
+}
diff --git a/qualys.go b/qualys.go
new file mode 100644
index 0000000..940431c
--- /dev/null
+++ b/qualys.go
@@ -0,0 +1,104 @@
+package main
+
+/*
+
+qualys - This file includes all of the logic necessary to interact with the
+go-qualys library. This is extrapolated out so that a QualysInterface
+interface can be passed to functions. Doing this allows testing by mock
+classes to be created that can be passed to functions.
+
+Since this is a wrapper around the go-qualys library, this does not need
+testing.
+
+*/
+
+import (
+ "errors"
+ "fmt"
+ "log"
+ "net/http"
+ "net/url"
+
+ "git.openstack.org/openstack/osel/qualys"
+)
+
+// QualysActioner is an interface for an QualysActions class. Having
+// this as an interface allows us to pass in a dummy class for testing that
+// just returns mocked data.
+type QualysActioner interface {
+ InitiateScan([]string) (string, error)
+ DropIPv6() bool
+}
+
+// QualysActions is a class that handles all interactions directly with Qualys.
+// See the comment on QualysActioner for rationale.
+type QualysActions struct {
+ Options QualysOptions
+}
+
+// QualysOptions is a class to convey all of the configurable options for the
+// QualysActions class.
+type QualysOptions struct {
+ DropIPv6 bool
+ MinRemaining int
+ ProxyURL *url.URL
+ Password string
+ QualysURL *url.URL
+ ScanOptionName string
+ UserName string
+}
+
+// InitiateScan is the main method for the QualysActioner class, it
+// makes a call to the Qualys API to start a scan and harvests a scan ID, and
+// an optional error string if there is a problem contacting Qualys.
+func (s *QualysActions) InitiateScan(targetIPAddresses []string) (string, error) {
+ var err error
+
+ // create client with proxy so the qualys service can be accessed
+ qualysCreds := qualys.Credentials{
+ Username: s.Options.UserName,
+ Password: s.Options.Password,
+ }
+ c, err := qualys.NewClient(&http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(s.Options.ProxyURL)}}, &qualysCreds)
+ if err != nil {
+ return "", err
+ }
+ c.BaseURL = s.Options.QualysURL
+
+ // create the options
+ opts := qualys.LaunchScanOptions{
+ ScanTitle: "osel",
+ ScannerName: "External",
+ OptionTitle: s.Options.ScanOptionName,
+ IP: targetIPAddresses,
+ }
+
+ // launch the request
+ launchScanResponse, err := c.LaunchScan(&opts)
+ if err != nil {
+ return "", err
+ }
+
+ // process the request response
+ scanID := launchScanResponse.ScanReference
+ remainingQualysRequests := launchScanResponse.RateLimitations.Remaining
+ allowedQualysRequests := launchScanResponse.RateLimitations.Limit
+ if Debug {
+ log.Printf("Qualys Rate Limit: %d of %d total requests remaining, concurrency of %d out of %d, %d seconds remaining in limit window and %d seconds until a request can be made again\n",
+ remainingQualysRequests, allowedQualysRequests, launchScanResponse.RateLimitations.CurrentConcurrency,
+ launchScanResponse.RateLimitations.ConcurrencyLimit, launchScanResponse.RateLimitations.LimitWindow, launchScanResponse.RateLimitations.WaitingPeriod)
+ }
+ if launchScanResponse.Text != "" {
+ err = errors.New(launchScanResponse.Text)
+ }
+ if remainingQualysRequests <= s.Options.MinRemaining {
+ err = fmt.Errorf("halting Qualys processing! Only %d Qualys calls remain out of a total of %d. Waiting for %d seconds before resuming", remainingQualysRequests,
+ allowedQualysRequests, launchScanResponse.RateLimitations.LimitWindow)
+ }
+ return scanID, err
+}
+
+// DropIPv6 is an accessor method to allow other code to make decisions based on whether this flag is enabled.
+func (s *QualysActions) DropIPv6() bool {
+ return s.Options.DropIPv6
+}
diff --git a/qualys/assets.go b/qualys/assets.go
new file mode 100644
index 0000000..ad2d50a
--- /dev/null
+++ b/qualys/assets.go
@@ -0,0 +1,170 @@
+package qualys
+
+import (
+ "bytes"
+ "fmt"
+ "net"
+ "strings"
+)
+
+const (
+ assetsBasePath = "asset"
+ groupsBasePath = "group"
+)
+
+// AssetsService is an interface for interfacing with the Assets
+// endpoints of the Qualys API
+type AssetsService interface {
+ ListAssetGroups(*ListAssetGroupOptions) ([]AssetGroup, *Response, error)
+ GetAssetGroupByID(groupID string) (*AssetGroup, *Response, error)
+ AddIPsToGroup(*AddIPsToGroupOptions) (*Response, error)
+}
+
+// AssetsServiceOp handles communication with the asset related methods of the
+// Qualys API.
+type AssetsServiceOp struct {
+ client *Client
+}
+
+var _ AssetsService = &AssetsServiceOp{}
+
+// AssetGroup represents a Qualys HostGroup
+type AssetGroup struct {
+ ID string `xml:"ID"`
+ Title string `xml:"TITLE"`
+ OwnerUserID string `xml:"OWNER_USER_ID"`
+ OwnerUnitID string `xml:"OWNER_UNIT_ID"`
+ IPs AssetGroupIPs `xml:"IP_SET"`
+}
+
+// AssetGroupIPs represents one or more IP addresses assigned to the AssetGroup
+type AssetGroupIPs struct {
+ IPs []string `xml:"IP"`
+ IPRanges []string `xml:"IP_RANGE"`
+}
+
+type ipRange struct {
+ Min net.IP
+ Max net.IP
+}
+
+func newIPRange(rangeString string) *ipRange {
+ var r = strings.Split(rangeString, "-")
+ return &ipRange{Min: net.ParseIP(r[0]), Max: net.ParseIP(r[1])}
+}
+
+func (ip *ipRange) Contains(ipString string) bool {
+ var myIP = net.ParseIP(ipString)
+ if bytes.Compare(myIP, ip.Min) >= 0 && bytes.Compare(myIP, ip.Max) <= 0 {
+ return true
+ }
+ return false
+}
+
+// ContainsIP returns true when the AssetGroupIPs matches the provided IP
+func (agp *AssetGroupIPs) ContainsIP(ip string) bool {
+ if containsString(agp.IPs, ip) {
+ return true
+ }
+ if agp.IPRanges != nil && len(agp.IPRanges) > 0 {
+ for _, ipRange := range agp.IPRanges {
+ if newIPRange(ipRange).Contains(ip) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// ContainsIP returns true when the AssetGroup has any assets matching the provided IP
+func (ag *AssetGroup) ContainsIP(ip string) bool {
+ return ag.IPs.ContainsIP(ip)
+}
+
+type assetGroupsRoot struct {
+ AssetGroups []AssetGroup `xml:"RESPONSE>ASSET_GROUP_LIST>ASSET_GROUP"`
+}
+
+// AssetGroupUpdateRequest represents a request to update a group
+type AssetGroupUpdateRequest struct {
+}
+
+// ListAssetGroupOptions represents the AssetGroup retrieval options
+type ListAssetGroupOptions struct {
+ Ids []string `url:"ids,comma,omitempty"`
+ Action string `url:"action,omitempty"`
+}
+
+// AddIPsToGroupOptions represents the update request for an AssetGroup
+type AddIPsToGroupOptions struct {
+ GroupID string `url:"id,omitempty"`
+ IPs []string `url:"add_ips,comma,omitempty"`
+}
+
+// ListAssetGroups retrieves a list of AssetGroups
+func (s *AssetsServiceOp) ListAssetGroups(opt *ListAssetGroupOptions) ([]AssetGroup, *Response, error) {
+ return s.listAssetGroups(opt)
+}
+
+// GetAssetGroupByID retrieves an AssetGroup by id.
+func (s *AssetsServiceOp) GetAssetGroupByID(groupID string) (*AssetGroup, *Response, error) {
+ return s.getAssetGroup(groupID)
+}
+
+// AddIPsToGroup adds the IPs in AddIPsToGroupOptions to the AssetGroup
+func (s *AssetsServiceOp) AddIPsToGroup(opt *AddIPsToGroupOptions) (*Response, error) {
+ return s.addIPsToGroup(opt)
+}
+
+func (s *AssetsServiceOp) getAssetGroup(groupID string) (*AssetGroup, *Response, error) {
+ opts := ListAssetGroupOptions{Ids: []string{groupID}}
+ groups, response, err := s.listAssetGroups(&opts)
+ if err != nil {
+ return nil, response, err
+ }
+ if len(groups) == 0 {
+ return nil, response, nil
+ }
+ return &groups[0], response, nil
+}
+
+func (s *AssetsServiceOp) addIPsToGroup(opt *AddIPsToGroupOptions) (*Response, error) {
+ path := fmt.Sprintf("%s/%s/?action=edit", assetsBasePath, groupsBasePath)
+
+ req, err := s.client.NewRequest("POST", path, opt)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := s.client.MakeRequest(req, nil)
+ if err != nil {
+ return resp, err
+ }
+ return resp, err
+}
+
+// Helper method for listing asset groups
+func (s *AssetsServiceOp) listAssetGroups(listOpt *ListAssetGroupOptions) ([]AssetGroup, *Response, error) {
+ path := fmt.Sprintf("%s/%s/", assetsBasePath, groupsBasePath)
+ if listOpt == nil {
+ listOpt = &ListAssetGroupOptions{}
+ }
+ listOpt.Action = "list"
+ path, err := addURLParameters(path, listOpt)
+
+ if err != nil {
+ return nil, nil, err
+ }
+
+ req, err := s.client.NewRequest("GET", path, nil)
+ if err != nil {
+ return nil, nil, err
+ }
+
+ root := new(assetGroupsRoot)
+ resp, err := s.client.MakeRequest(req, root)
+ if err != nil {
+ return nil, resp, err
+ }
+ return root.AssetGroups, resp, err
+}
diff --git a/qualys/assets_test.go b/qualys/assets_test.go
new file mode 100644
index 0000000..4e892d0
--- /dev/null
+++ b/qualys/assets_test.go
@@ -0,0 +1,251 @@
+package qualys
+
+import (
+ "fmt"
+ "net/http"
+ "reflect"
+ "testing"
+)
+
+func TestListAssetGroups(t *testing.T) {
+
+ cases := []struct {
+ name string
+ response string
+ expected []AssetGroup
+ opts *ListAssetGroupOptions
+ isErr bool
+ }{
+ {
+ name: "ListAssetGroups - single item, without list options",
+ response: assetGroupsXMLSingleGroup,
+ expected: []AssetGroup{
+ {
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.1.1.1", "10.10.10.11"},
+ IPRanges: nil,
+ },
+ },
+ },
+ opts: nil,
+ },
+ {
+ name: "ListAssetGroups - single item, with list options",
+ response: assetGroupsXMLSingleGroup,
+ expected: []AssetGroup{
+ {
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.1.1.1", "10.10.10.11"},
+ IPRanges: nil,
+ },
+ },
+ },
+ opts: &ListAssetGroupOptions{Ids: []string{}},
+ },
+ {
+ name: "ListAssetGroups - multi item",
+ response: assetGroupsXMLMultiGroups,
+ expected: []AssetGroup{
+ {ID: "1759734", Title: "AG - New"},
+ {ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.10.10.14"},
+ IPRanges: []string{"10.10.10.3-10.10.10.6"},
+ },
+ },
+ },
+ opts: &ListAssetGroupOptions{Ids: []string{"1", "2"}},
+ },
+ }
+
+ for _, c := range cases {
+ setup()
+ defer teardown()
+ mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
+ testMethod(t, r, "GET")
+ fmt.Fprint(w, c.response)
+ })
+
+ assetGroups, _, err := client.Assets.ListAssetGroups(c.opts)
+ if err != nil {
+ t.Errorf("Assets.ListAssetGroups returned error: %v", err)
+ }
+
+ if !reflect.DeepEqual(assetGroups, c.expected) {
+ t.Errorf("Assets.ListAssetGroups case: %s returned %+v, expected %+v", c.name, assetGroups, c.expected)
+ }
+ }
+}
+
+func TestGetAssetGroupByID(t *testing.T) {
+ setup()
+ defer teardown()
+
+ mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
+ testMethod(t, r, "GET")
+ fmt.Fprint(w, assetGroupsXMLSingleGroup)
+ })
+
+ groupID := "1759735"
+
+ assetGroup, _, err := client.Assets.GetAssetGroupByID(groupID)
+ if err != nil {
+ t.Errorf("Assets.GetAssetGroupByID(%s) returned error: %v", groupID, err)
+ }
+
+ expected := &AssetGroup{
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.1.1.1", "10.10.10.11"},
+ IPRanges: nil,
+ },
+ }
+ if !reflect.DeepEqual(assetGroup, expected) {
+ t.Errorf("Assets.GetAssetGroupByID(%s) returned %+v, expected %+v", groupID, assetGroup, expected)
+ }
+}
+
+func TestAddIPsToGroup(t *testing.T) {
+ setup()
+ defer teardown()
+
+ groupID := "1759735"
+ ip := "10.10.10.10"
+
+ mux.HandleFunc("/asset/group/", func(w http.ResponseWriter, r *http.Request) {
+ testMethod(t, r, "POST")
+ if r.FormValue("add_ips") != ip {
+ t.Errorf("Request form data did not include the correct IP")
+ }
+ if r.FormValue("id") != groupID {
+ t.Errorf("Request form data did not include the correct asset group ID")
+ }
+ fmt.Fprint(w, assetGroupsAddIPsResponse)
+ })
+ opts := &AddIPsToGroupOptions{
+ GroupID: groupID,
+ IPs: []string{ip},
+ }
+
+ _, err := client.Assets.AddIPsToGroup(opts)
+ if err != nil {
+ t.Errorf("Assets.AddIPsToGroup returned error: %v", err)
+ }
+}
+
+func TestAssetGroupContainsIP(t *testing.T) {
+ cases := []struct {
+ name string
+ ip string
+ group *AssetGroup
+ expected bool
+ }{
+ {
+ name: "AssetGroup.ContainsIP - nil",
+ ip: "10.1.1.1",
+ group: &AssetGroup{ID: "1759735", Title: "AG - Elastic Cloud Dynamic Perimeter"},
+ expected: false,
+ },
+ {
+ name: "AssetGroup.ContainsIP - empty",
+ ip: "10.1.1.1",
+ group: &AssetGroup{
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{}},
+ expected: false,
+ },
+ {
+ name: "AssetGroup.ContainsIP - single item list",
+ ip: "10.1.1.1",
+ group: &AssetGroup{
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.1.1.1"},
+ IPRanges: []string{},
+ },
+ },
+ expected: true,
+ },
+ {
+ name: "AssetGroup.ContainsIP - multi item list",
+ ip: "10.1.1.1",
+ group: &AssetGroup{
+ ID: "1759735",
+ Title: "AG - Elastic Cloud Dynamic Perimeter",
+ IPs: AssetGroupIPs{
+ IPs: []string{"10.1.1.1"},
+ IPRanges: []string{"10.10.1.1-10.10.10.10"},
+ },
+ },
+ expected: true,
+ },
+ }
+ for _, c := range cases {
+ contains := c.group.ContainsIP(c.ip)
+ if contains != c.expected {
+ t.Errorf("%s - AssetGroup.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected)
+ }
+ }
+}
+
+func TestAssetGroupIPsContainsIP(t *testing.T) {
+ group := AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.10.3-10.10.10.6"}}
+
+ cases := []struct {
+ name string
+ ip string
+ group AssetGroupIPs
+ expected bool
+ }{
+ {
+ name: "AssetGroupIPs.ContainsIP - IP value match",
+ ip: "10.0.1.1",
+ group: group,
+ expected: true,
+ },
+ {
+ name: "AssetGroupIPs.ContainsIP - IP value no match",
+ ip: "192.0.1.1",
+ group: group,
+ expected: false,
+ },
+ {
+ name: "AssetGroupIPs.ContainsIP - IP Range value match",
+ ip: "10.10.10.4",
+ group: group,
+ expected: true,
+ },
+ {
+ name: "AssetGroupIPs.ContainsIP - IP Range value no match",
+ ip: "10.10.10.1",
+ group: group,
+ expected: false,
+ },
+ {
+ name: "AssetGroupIPs.ContainsIP - IP Range value match",
+ ip: "10.10.0.4",
+ group: AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.0.0-10.10.10.6"}},
+ expected: true,
+ },
+ {
+ name: "AssetGroupIPs.ContainsIP - IP Range value no match",
+ ip: "10.10.0.4",
+ group: AssetGroupIPs{IPs: []string{"10.0.1.1"}, IPRanges: []string{"10.10.1.3-10.10.10.6"}},
+ expected: false,
+ },
+ }
+
+ for _, c := range cases {
+ contains := c.group.ContainsIP(c.ip)
+ if contains != c.expected {
+ t.Errorf("%s - AssetGroupIPs.ContainsIP(%s) returned %v, expected %v", c.name, c.ip, contains, c.expected)
+ }
+ }
+}
diff --git a/qualys/assets_xml_fixtures_test.go b/qualys/assets_xml_fixtures_test.go
new file mode 100644
index 0000000..55f2b3a
--- /dev/null
+++ b/qualys/assets_xml_fixtures_test.go
@@ -0,0 +1,68 @@
+package qualys
+
+const (
+ assetGroupsXMLSingleGroup = `
+
+
+
+
+ 2016-10-05T19:00:22Z
+
+
+ 1759735
+ <![CDATA[AG - Elastic Cloud Dynamic Perimeter]]>
+
+ 10.1.1.1
+ 10.10.10.11
+
+
+
+
+
+
+ `
+
+ assetGroupsXMLMultiGroups = `
+
+
+
+
+ 2016-10-05T19:00:22Z
+
+
+ 1759734
+ <![CDATA[AG - New]]>
+ 105102
+ 105102
+
+
+ 1759735
+ <![CDATA[AG - Elastic Cloud Dynamic Perimeter]]>
+
+ 10.10.10.3-10.10.10.6
+ 10.10.10.14
+
+
+
+
+
+
+ `
+
+ assetGroupsAddIPsResponse = `
+
+
+
+
+ 2016-10-12T14:16:22Z
+ Asset Group Updated Successfully
+
+ -
+ ID
+ 1759735
+
+
+
+
+ `
+)
diff --git a/qualys/client.go b/qualys/client.go
new file mode 100644
index 0000000..dc9241e
--- /dev/null
+++ b/qualys/client.go
@@ -0,0 +1,130 @@
+package qualys
+
+import (
+ "fmt"
+ "net/http"
+ "net/url"
+)
+
+const (
+ libraryVersion = "0.1.0"
+ defaultBaseURL = "https://qualysapi.qualys.com/api/2.0/fo/"
+ userAgent = "go-qualys"
+ mediaType = "application/xml"
+
+ headerUserAgent = "X-Requested-With"
+ headerRateLimit = "X-RateLimit-Limit"
+ headerRateLimitWindow = "X-RateLimit-Window-Sec"
+ headerRateRemaining = "X-RateLimit-Remaining"
+ headerRateLimitWait = "X-RateLimit-ToWait-Sec"
+ headerConcurrencyLimit = "X-Concurrency-Limit-Limit"
+ headerConcurrencyLimitRunning = "X-Concurrency-Limit-Running"
+)
+
+// Client for Qualys API
+type Client struct {
+ // Credentials used to authenticate to the Qualys API
+ Credentials *Credentials
+
+ // HTTP client used to communicate with the Qualys API
+ client *http.Client
+
+ // Base URL for API requests.
+ BaseURL *url.URL
+
+ // User agent for client
+ UserAgent string
+
+ // Rate contains the current rate limit for the client as determined by the most recent
+ // API call.
+ Rate Rate
+
+ // Services used for communicating with the API
+ Assets AssetsService
+}
+
+// Rate contains the rate limit for the current client.
+type Rate struct {
+ // The number of requests within the limit window of seconds the client is allowed
+ Limit int
+
+ // The number of seconds remaining in the limit window
+ LimitWindow int
+
+ // The number of remaining requests the client can make during the limit window period
+ Remaining int
+
+ // The number of seconds to wait before requests can be made again -- headerRateLimitWait
+ WaitingPeriod int
+
+ // The number of API calls permitted to be executed concurrrently
+ ConcurrencyLimit int
+
+ // The number of API calls currently running
+ CurrentConcurrency int
+}
+
+// Credentials holds the credentials and endpoint for the Qualys Client
+type Credentials struct {
+ Username string
+ Password string
+}
+
+// ClientOpt are options for New.
+type ClientOpt func(*Client) error
+
+// New returns a new API client instance.
+func New(httpClient *http.Client, credentials *Credentials, opts ...ClientOpt) (*Client, error) {
+ c, err := NewClient(httpClient, credentials)
+ if err != nil {
+ return nil, err
+ }
+ for _, opt := range opts {
+ if err := opt(c); err != nil {
+ return nil, err
+ }
+ }
+
+ return c, nil
+}
+
+// NewClient returns a new Qualys API client.
+func NewClient(httpClient *http.Client, credentials *Credentials) (*Client, error) {
+ if httpClient == nil {
+ httpClient = http.DefaultClient
+ }
+ if credentials == nil || credentials.Username == "" || credentials.Password == "" {
+ return nil, fmt.Errorf("Credentials must be provided")
+ }
+
+ baseURL, err := url.Parse(defaultBaseURL)
+ if err != nil {
+ return nil, err
+ }
+
+ c := &Client{client: httpClient, Credentials: credentials, BaseURL: baseURL, UserAgent: userAgent}
+
+ c.Assets = &AssetsServiceOp{client: c}
+ return c, nil
+}
+
+// SetBaseURL is a client option for setting the base URL.
+func SetBaseURL(bu string) ClientOpt {
+ return func(c *Client) error {
+ u, err := url.Parse(bu)
+ if err != nil {
+ return err
+ }
+
+ c.BaseURL = u
+ return nil
+ }
+}
+
+// SetUserAgent is a client option for setting the user agent.
+func SetUserAgent(ua string) ClientOpt {
+ return func(c *Client) error {
+ c.UserAgent = fmt.Sprintf("%s+%s", ua, c.UserAgent)
+ return nil
+ }
+}
diff --git a/qualys/client_test.go b/qualys/client_test.go
new file mode 100644
index 0000000..d4de53f
--- /dev/null
+++ b/qualys/client_test.go
@@ -0,0 +1,207 @@
+package qualys
+
+import (
+ "bytes"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "reflect"
+ "testing"
+)
+
+var (
+ mux *http.ServeMux
+
+ client *Client
+
+ server *httptest.Server
+
+ creds *Credentials
+)
+
+func setup() {
+ mux = http.NewServeMux()
+ server = httptest.NewServer(mux)
+ creds = &Credentials{Username: "bogus", Password: "bogus"}
+ client, _ = NewClient(nil, creds)
+ url, _ := url.Parse(server.URL)
+ client.BaseURL = url
+}
+
+func teardown() {
+ server.Close()
+}
+
+func testMethod(t *testing.T, r *http.Request, expected string) {
+ if expected != r.Method {
+ t.Errorf("Request method = %v, expected %v", r.Method, expected)
+ }
+}
+
+type values map[string]string
+
+func testFormValues(t *testing.T, r *http.Request, values values) {
+ expected := url.Values{}
+ for k, v := range values {
+ expected.Add(k, v)
+ }
+
+ err := r.ParseForm()
+ if err != nil {
+ t.Fatalf("parseForm(): %v", err)
+ }
+
+ if !reflect.DeepEqual(expected, r.Form) {
+ t.Errorf("Request parameters = %v, expected %v", r.Form, expected)
+ }
+}
+
+func testURLParseError(t *testing.T, err error) {
+ if err == nil {
+ t.Errorf("Expected error to be returned")
+ }
+ if err, ok := err.(*url.Error); !ok || err.Op != "parse" {
+ t.Errorf("Expected URL parse error, got %+v", err)
+ }
+}
+
+func testClientDefaultBaseURL(t *testing.T, c *Client) {
+ if c.BaseURL == nil || c.BaseURL.String() != defaultBaseURL {
+ t.Errorf("NewClient BaseURL = %v, expected %v", c.BaseURL, defaultBaseURL)
+ }
+}
+
+func testClientDefaultUserAgent(t *testing.T, c *Client) {
+ if c.UserAgent != userAgent {
+ t.Errorf("NewClick UserAgent = %v, expected %v", c.UserAgent, userAgent)
+ }
+}
+
+func testClientDefaults(t *testing.T, c *Client) {
+ testClientDefaultBaseURL(t, c)
+ testClientDefaultUserAgent(t, c)
+}
+
+func TestNewClient(t *testing.T) {
+ c, _ := NewClient(nil, creds)
+ testClientDefaults(t, c)
+}
+
+func TestNew(t *testing.T) {
+ c, err := New(nil, creds)
+
+ if err != nil {
+ t.Fatalf("New(): %v", err)
+ }
+ testClientDefaults(t, c)
+}
+
+func TestNewClientWithoutCredentials(t *testing.T) {
+ _, err := NewClient(nil, nil)
+
+ if err == nil {
+ t.Errorf("NewClient() expected error when Credentials are not set")
+ }
+}
+
+func TestCustomUserAgent(t *testing.T) {
+ c, err := New(nil, creds, SetUserAgent("testing"))
+
+ if err != nil {
+ t.Fatalf("New() unexpected error: %v", err)
+ }
+
+ expected := fmt.Sprintf("%s+%s", "testing", userAgent)
+ if got := c.UserAgent; got != expected {
+ t.Errorf("New() UserAgent = %s; expected %s", got, expected)
+ }
+}
+
+func TestAddURLParameters(t *testing.T) {
+ cases := []struct {
+ name string
+ path string
+ expected string
+ opts *ListAssetGroupOptions
+ isErr bool
+ }{
+ {
+ name: "addURLParameters",
+ path: "/asset/group/",
+ expected: "/asset/group/?ids=1",
+ opts: &ListAssetGroupOptions{Ids: []string{"1"}},
+ isErr: false,
+ },
+ {
+ name: "addURLParameters with slice parameter",
+ path: "/asset/group/",
+ expected: "/asset/group/?ids=1,2",
+ opts: &ListAssetGroupOptions{Ids: []string{"1", "2"}},
+ isErr: false,
+ },
+ }
+
+ for _, c := range cases {
+ got, err := addURLParameters(c.path, c.opts)
+ if c.isErr && err == nil {
+ t.Errorf("%q expected error but none was encountered", c.name)
+ continue
+ }
+
+ if !c.isErr && err != nil {
+ t.Errorf("%q unexpected error: %v", c.name, err)
+ continue
+ }
+
+ gotURL, err := url.Parse(got)
+ if err != nil {
+ t.Errorf("%q unable to parse returned URL", c.name)
+ continue
+ }
+
+ expectedURL, err := url.Parse(c.expected)
+ if err != nil {
+ t.Errorf("%q unable to parse expected URL", c.name)
+ continue
+ }
+
+ if g, e := gotURL.Path, expectedURL.Path; g != e {
+ t.Errorf("%q path = %q; expected %q", c.name, g, e)
+ continue
+ }
+
+ if g, e := gotURL.Query(), expectedURL.Query(); !reflect.DeepEqual(g, e) {
+ t.Errorf("%q query = %#v; expected %#v", c.name, g, e)
+ continue
+ }
+ }
+}
+
+func TestFormPostBody(t *testing.T) {
+ cases := []struct {
+ name string
+ expected string
+ opts *AddIPsToGroupOptions
+ }{
+ {
+ name: "formPostBody single IP",
+ expected: "add_ips=10.10.10.10&id=1234",
+ opts: &AddIPsToGroupOptions{GroupID: "1234", IPs: []string{"10.10.10.10"}},
+ },
+ {
+ name: "formPostBody multi IP",
+ expected: "add_ips=10.10.10.10%2C10.10.10.11&id=1234",
+ opts: &AddIPsToGroupOptions{GroupID: "1234", IPs: []string{"10.10.10.10", "10.10.10.11"}},
+ },
+ }
+ for _, c := range cases {
+ got, _ := formPostBody(c.opts)
+ buf := new(bytes.Buffer)
+ buf.ReadFrom(got)
+ bodyString := buf.String()
+ if c.expected != bodyString {
+ t.Errorf("%q expected %s but got %s", c.name, c.expected, bodyString)
+ }
+ }
+}
diff --git a/qualys/requests.go b/qualys/requests.go
new file mode 100644
index 0000000..316241d
--- /dev/null
+++ b/qualys/requests.go
@@ -0,0 +1,137 @@
+package qualys // import "git.openstack.org/openstack/osel/qualys"
+
+import (
+ "bytes"
+ "encoding/xml"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "strconv"
+
+ "github.com/google/go-querystring/query"
+)
+
+// Response is a Qualys API response. This wraps the standard http.Response returned from Qualys.
+type Response struct {
+ *http.Response
+
+ Rate
+}
+
+// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the
+// BaseURL of the Client. Relative URLS should always be specified without a preceding slash. If specified, the
+// value pointed to by body is form encoded and included as the request body.
+func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
+ rel, err := url.Parse(urlStr)
+ if err != nil {
+ return nil, err
+ }
+
+ u := c.BaseURL.ResolveReference(rel)
+ buf := new(bytes.Buffer)
+
+ if method == http.MethodPost {
+ buf, err = formPostBody(body)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ req, err := http.NewRequest(method, u.String(), buf)
+ if err != nil {
+ return nil, err
+ }
+
+ if method == http.MethodPost {
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ }
+ req.SetBasicAuth(c.Credentials.Username, c.Credentials.Password)
+ req.Header.Set(headerUserAgent, userAgent)
+ return req, nil
+}
+
+// newResponse creates a new Response for the provided http.Response
+func newResponse(r *http.Response) *Response {
+ response := Response{Response: r}
+ response.populateRate()
+
+ return &response
+}
+
+// populateRate parses the rate related headers and populates the response Rate.
+func (r *Response) populateRate() {
+ // TODO - deal with the rest of the headers
+ if limit := r.Header.Get(headerRateLimit); limit != "" {
+ r.Rate.Limit, _ = strconv.Atoi(limit)
+ }
+ if remaining := r.Header.Get(headerRateRemaining); remaining != "" {
+ r.Rate.Remaining, _ = strconv.Atoi(remaining)
+ }
+ if rateLimitWindow := r.Header.Get(headerRateLimitWindow); rateLimitWindow != "" {
+ r.Rate.LimitWindow, _ = strconv.Atoi(rateLimitWindow)
+ }
+ if waitingPeriod := r.Header.Get(headerRateLimitWait); waitingPeriod != "" {
+ r.Rate.WaitingPeriod, _ = strconv.Atoi(waitingPeriod)
+ }
+ if concurrencyLimit := r.Header.Get(headerConcurrencyLimit); concurrencyLimit != "" {
+ r.Rate.ConcurrencyLimit, _ = strconv.Atoi(concurrencyLimit)
+ }
+ if runningConcurrencyLimit := r.Header.Get(headerConcurrencyLimitRunning); runningConcurrencyLimit != "" {
+ r.Rate.CurrentConcurrency, _ = strconv.Atoi(runningConcurrencyLimit)
+ }
+}
+
+// MakeRequest sends an API request and returns the API response. The API response is XML decoded and stored in the value
+// pointed to by v, or returned as an error if an API error has occurred..
+func (c *Client) MakeRequest(req *http.Request, v interface{}) (*Response, error) {
+ resp, err := c.client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ defer func() {
+ if rerr := resp.Body.Close(); err == nil {
+ err = rerr
+ }
+ }()
+
+ response := newResponse(resp)
+ c.Rate = response.Rate
+
+ err = CheckResponse(resp)
+ if err != nil {
+ return response, err
+ }
+
+ bodyContents, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return response, err
+ }
+
+ if v != nil {
+ err := xml.Unmarshal(bodyContents, v)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return response, err
+}
+
+// CheckResponse checks the API response for errors, and returns them if present. A response is considered an
+// error if it has a status code outside the 200 range.
+func CheckResponse(r *http.Response) error {
+ if c := r.StatusCode; c >= 200 && c <= 299 {
+ return nil
+ }
+ return fmt.Errorf("Response status is: %v", r.StatusCode)
+}
+
+func formPostBody(opt interface{}) (*bytes.Buffer, error) {
+ vals, err := query.Values(opt)
+ if err != nil {
+ return nil, err
+ }
+ return bytes.NewBufferString(vals.Encode()), nil
+}
diff --git a/qualys/scans.go b/qualys/scans.go
new file mode 100644
index 0000000..6b072bd
--- /dev/null
+++ b/qualys/scans.go
@@ -0,0 +1,186 @@
+package qualys
+
+import (
+ "encoding/xml"
+ "errors"
+ "net/http"
+ "strings"
+)
+
+var ErrMalformedResponse error = errors.New("malformed xml response from server")
+
+// LaunchScanResponse is the expected response for a scan launch
+// xml tag: simple_return
+// DateTime is the date and time the scan was issued
+// It includes a key/value map of pertinent information.
+type LaunchScanResponse struct {
+ XMLName xml.Name `xml:"SIMPLE_RETURN"`
+ Value []string `xml:"RESPONSE>ITEM_LIST>ITEM>VALUE"`
+ Key []string `xml:"RESPONSE>ITEM_LIST>ITEM>KEY"`
+ Datetime string `xml:"RESPONSE>DATETIME"`
+ Text string `xml:"RESPONSE>TEXT"`
+
+ ScanReference string
+
+ RateLimitations Rate
+}
+
+type LaunchScanOptions struct {
+ ScanTitle string `url:"scan_title"`
+ OptionID int64 `url:"option_id,omitempty"`
+ OptionTitle string `url:"option_title"`
+ ScannerName string `url:"iscanner_name"`
+ IP []string `url:"ip"`
+ Action string `url:"action"`
+}
+
+// ScanLaunch will try and run a scan on demand
+// has certain parameters that must be filled in
+func (client *Client) LaunchScan(options *LaunchScanOptions) (*LaunchScanResponse, error) {
+ options.Action = "launch"
+
+ urlString, err := addURLParameters(client.BaseURL.String(), options)
+
+ if err != nil {
+ return nil, err
+ }
+
+ // i don't think there's any header data here?
+ req, err := client.NewRequest(http.MethodPost, urlString, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ var resp LaunchScanResponse
+
+ response, err := client.MakeRequest(req, &resp)
+
+ if err != nil {
+ return nil, err
+ }
+
+ resp.RateLimitations = response.Rate
+
+ for key, val := range resp.Key {
+ if strings.ToUpper(val) == "REFERENCE" {
+ // should check len first
+ if len(resp.Value) > key {
+ resp.ScanReference = resp.Value[key]
+ } else {
+ return nil, ErrMalformedResponse
+ }
+ }
+ }
+
+ return &resp, nil
+}
+
+type PollScanResponse struct {
+ XMLName xml.Name `xml:"SCAN_LIST_OUTPUT"`
+ Datetime string `xml:"RESPONSE>DATETIME"`
+ Processing_priority string `xml:"RESPONSE>SCAN_LIST>SCAN>PROCESSING_PRIORITY"`
+ Processed string `xml:"RESPONSE>SCAN_LIST>SCAN>PROCESSED"`
+ Launch_datetime string `xml:"RESPONSE>SCAN_LIST>SCAN>LAUNCH_DATETIME"`
+ Target string `xml:"RESPONSE>SCAN_LIST>SCAN>TARGET"`
+ Type string `xml:"RESPONSE>SCAN_LIST>SCAN>TYPE"`
+ Title string `xml:"RESPONSE>SCAN_LIST>SCAN>TITLE"`
+ User_login string `xml:"RESPONSE>SCAN_LIST>SCAN>USER_LOGIN"`
+ Ref string `xml:"RESPONSE>SCAN_LIST>SCAN>REF"`
+ Duration string `xml:"RESPONSE>SCAN_LIST>SCAN>DURATION"`
+ State string `xml:"RESPONSE>SCAN_LIST>SCAN>STATUS>STATE"`
+}
+
+type PollScanOptions struct {
+ Action string `url:"action"` // list
+ ScanRef string `url:"scan_ref"`
+}
+
+func (client *Client) PollScanResults(options *PollScanOptions) (*PollScanResponse, error) {
+ options.Action = "list"
+
+ urlString, err := addURLParameters(client.BaseURL.String(), options)
+
+ if err != nil {
+ return nil, err
+ }
+
+ req, err := client.NewRequest(http.MethodGet, urlString, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ var resp PollScanResponse
+
+ _, requestError := client.MakeRequest(req, &resp)
+
+ if requestError != nil {
+ return nil, requestError
+ }
+
+ return &resp, nil
+}
+
+type CompletedScanResponse struct {
+ XMLName xml.Name `xml:"SIMPLE_RETURN"` // the actual outp was supposed to be: SCAN_LIST_OUTPUT?
+ City string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>CITY"`
+ Key []key `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>KEY"`
+ Name string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>NAME"`
+ ZIPCode string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>ZIP_CODE"`
+ Address string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>ADDRESS"`
+ NameUserInfoHeaderComplianceScanResponse string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>USER_INFO>NAME"`
+ Username string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>USER_INFO>USERNAME"`
+ Role string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>USER_INFO>ROLE"`
+ Hosts string `xml:"RESPONSE>COMPLIANCE_SCAN>APPENDIX>TARGET_DISTRIBUTION>SCANNER>HOSTS"`
+ Type string `xml:"RESPONSE>COMPLIANCE_SCAN>APPENDIX>AUTHENTICATION>AUTH>TYPE"`
+ Datetime string `xml:"RESPONSE>DATETIME"`
+ IP string `xml:"RESPONSE>COMPLIANCE_SCAN>APPENDIX>AUTHENTICATION>AUTH>SUCCESS>IP"`
+ State string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>STATE"`
+ Country string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>COMPANY_INFO>COUNTRY"`
+ HostsScanned string `xml:"RESPONSE>COMPLIANCE_SCAN>APPENDIX>TARGET_HOSTS>HOSTS_SCANNED"`
+ NameScannerTargetDistributionAppendixComplianceScanResponse string `xml:"RESPONSE>COMPLIANCE_SCAN>APPENDIX>TARGET_DISTRIBUTION>SCANNER>NAME"`
+ GenerationDateTime string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>GENERATION_DATETIME"`
+ NameHeaderComplianceScanResponse string `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>NAME"`
+ OptionProfileTitle optionProfileTitle `xml:"RESPONSE>COMPLIANCE_SCAN>HEADER>OPTION_PROFILE>OPTION_PROFILE_TITLE"`
+}
+
+type key struct {
+ XMLName xml.Name `xml:"KEY"`
+ Value string `xml:"value,attr"`
+ Text string `xml:",chardata"`
+}
+type optionProfileTitle struct {
+ XMLName xml.Name `xml:"OPTION_PROFILE_TITLE"`
+ Option_profile_default string `xml:"option_profile_default,attr"`
+ Text string `xml:",chardata"`
+}
+
+type CompletedScanOptions struct {
+ Action string `url:"action"` // fetch
+ ScanRef string `url:"scan_ref"`
+}
+
+func (client *Client) GetScanResults(options *CompletedScanOptions) (*CompletedScanResponse, error) {
+ options.Action = "fetch"
+
+ urlString, err := addURLParameters(client.BaseURL.String(), options)
+
+ if err != nil {
+ return nil, err
+ }
+
+ // i don't think there's any header data here?
+ req, err := client.NewRequest(http.MethodGet, urlString, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ var resp CompletedScanResponse
+
+ _, requestError := client.MakeRequest(req, &resp)
+
+ if requestError != nil {
+ return nil, requestError
+ }
+
+ return &resp, nil
+}
diff --git a/qualys/scans_test.go b/qualys/scans_test.go
new file mode 100644
index 0000000..b05ce2d
--- /dev/null
+++ b/qualys/scans_test.go
@@ -0,0 +1,71 @@
+package qualys
+
+import (
+ "net/http"
+ "net/url"
+ "testing"
+ "time"
+)
+
+var devCreds Credentials = Credentials{
+ Username: "cmcas_ae2",
+ Password: "D02debLYko",
+}
+
+func TestLiveScan(t *testing.T) {
+ // create client
+ c, clientErr := NewClient(&http.Client{}, &devCreds)
+
+ if clientErr != nil {
+ t.Error(clientErr)
+ }
+
+ if baseURL, err := url.Parse("https://qualysapi.qualys.com/api/2.0/fo/scan/"); err != nil {
+ t.Error(err)
+ } else {
+ c.BaseURL = baseURL
+ }
+
+ // create the options
+ opts := LaunchScanOptions{
+ ScanTitle: "hello_world",
+ ScannerName: "External",
+ // OptionID: 923922,
+ OptionTitle: "Elastic Cloud Option Profile with Password Guessing",
+ IP: []string{"96.119.99.178"},
+ }
+
+ // launch the request
+
+ launchScanResponse, err := c.LaunchScan(&opts)
+
+ if err != nil {
+ t.Error(err)
+ }
+
+ // not sure if necessary
+ time.Sleep(time.Minute * 1)
+
+ //time to poll the scan results
+ pollOpts := PollScanOptions{
+ ScanRef: launchScanResponse.ScanReference,
+ }
+
+ _, pollRespErr := c.PollScanResults(&pollOpts)
+
+ if pollRespErr != nil {
+ t.Error(pollRespErr)
+ }
+
+ // now need to keep polling until the results are all in...
+
+ resultsOptions := CompletedScanOptions{
+ ScanRef: launchScanResponse.ScanReference,
+ }
+
+ _, resultsRespErr := c.GetScanResults(&resultsOptions)
+
+ if resultsRespErr != nil {
+ t.Error(resultsRespErr)
+ }
+}
diff --git a/qualys/utils.go b/qualys/utils.go
new file mode 100644
index 0000000..d24169f
--- /dev/null
+++ b/qualys/utils.go
@@ -0,0 +1,44 @@
+package qualys
+
+import (
+ "net/url"
+ "reflect"
+
+ "github.com/google/go-querystring/query"
+)
+
+func containsString(strList []string, testStr string) bool {
+ for _, str := range strList {
+ if testStr == str {
+ return true
+ }
+ }
+ return false
+}
+
+func addURLParameters(urlString string, opt interface{}) (string, error) {
+ v := reflect.ValueOf(opt)
+
+ if v.Kind() == reflect.Ptr && v.IsNil() {
+ return urlString, nil
+ }
+
+ origURL, err := url.Parse(urlString)
+ if err != nil {
+ return urlString, err
+ }
+
+ origValues := origURL.Query()
+
+ newValues, err := query.Values(opt)
+ if err != nil {
+ return urlString, err
+ }
+
+ for k, v := range newValues {
+ origValues[k] = v
+ }
+
+ origURL.RawQuery = origValues.Encode()
+ return origURL.String(), nil
+}
diff --git a/qualys/xml/scan_launch.xml b/qualys/xml/scan_launch.xml
new file mode 100644
index 0000000..d93a29d
--- /dev/null
+++ b/qualys/xml/scan_launch.xml
@@ -0,0 +1,18 @@
+
+
+
+
+ 2017-02-02T14:28:35Z
+ New vm scan launched
+
+ -
+ ID
+ 26148615
+
+ -
+ REFERENCE
+ scan/1486045714.48615
+
+
+
+
\ No newline at end of file
diff --git a/qualys/xml/scan_list_output.xml b/qualys/xml/scan_list_output.xml
new file mode 100644
index 0000000..3ffa0e2
--- /dev/null
+++ b/qualys/xml/scan_list_output.xml
@@ -0,0 +1,23 @@
+
+
+
+
+ 2017-02-02T14:48:09Z
+
+
+ [scan/1486045714.48615]
+ API
+ <![CDATA[Elastic Cloud Automated Scan]]>
+ cmcas_at1
+ 2017-02-02T14:28:34Z
+ 00:08:44
+ 0 - No Priority
+ 1
+
+ Finished
+
+
+
+
+
+
\ No newline at end of file
diff --git a/qualys/xml/scan_results.xml b/qualys/xml/scan_results.xml
new file mode 100644
index 0000000..1114584
--- /dev/null
+++ b/qualys/xml/scan_results.xml
@@ -0,0 +1,81 @@
+
+
+
+
+ 2012-09-17T10:23:53Z
+
+
+
+
+
+
+ 2012-09-17T10:23:53Z
+
+
+
+
+
+
+
+
+
+
+
+ USERNAME
+ Manager
+
+ USERNAME
+
+ 2012-09-15T11:49:08Z
+
+ 10.10.10.29
+
+ 00:01:00
+
+ 10.10.21.122 (Scanner 6.6.28-1, Vulnerability Signatures 2.2.215-2)
+
+ 1
+ 1
+ Scheduled
+
+ File Integrity Monitoring: Enabled,
+ Scanned Ports: Standard Scan, Hosts to Scan in Parallel - External
+ Scanners: 15, Hosts to Scan in Parallel - Scanner Appliances: 30,
+ Total Processes to Run in Parallel: 10, HTTP Processes to Run in
+ Parallel: 10,
+ Packet (Burst) Delay: Medium, Intensity: Normal, Overall Performance:
+ Normal, ICMP Host Discovery, Ignore RST packets: Off, Ignore firewallgenerated
+ SYN-ACK packets: Off, Do not send ACK or SYN-ACK packets
+ during host discovery: Off
+
+ FINISHED
+
+
+
+
+
+
+
+
+ 10.10.10.29
+
+
+
+
+ 10.10.10.29
+
+
+
+
+ Windows
+
+ 10.10.10.29
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/qualys_mock_test.go b/qualys_mock_test.go
new file mode 100644
index 0000000..fb4bfd2
--- /dev/null
+++ b/qualys_mock_test.go
@@ -0,0 +1,36 @@
+package main
+
+import (
+ "log"
+ //"github.com/satori/go.uuid"
+)
+
+// QualysActions is a class that handles all interactions directly with Qualys.
+// See the comment on QualysActioner for rationale.
+type QualysTestActions struct {
+ testUUID string
+}
+
+// InitiateScan is the main method for the QualysActioner class, it
+// makes a call to the Qualys API to start a scan and harvests a scan ID, and
+// an optional error string if there is a problem contacting Qualys.
+func (s *QualysTestActions) InitiateScan(ipAddresses []string) (string, error) {
+ //testUUID = uuid.NewV4().String()
+ s.testUUID = `5fbf3cef-976e-475d-bd84-47ef23638a6b`
+ log.Printf("FAKE QUALYS SCAN: %s\n", s.testUUID)
+ return s.testUUID, nil
+}
+
+// GetTestScanID returns the fake UUID created in testing. This allows for
+// inspection of the UUID in unit tests.
+func (s *QualysTestActions) GetTestScanID() string {
+ return s.testUUID
+}
+
+func (s *QualysTestActions) DropIPv6() bool {
+ return false
+}
+
+func connectFakeQualys() *QualysTestActions {
+ return new(QualysTestActions)
+}
diff --git a/releasenotes/notes/initial-import-8cdbc214e8596521.yaml b/releasenotes/notes/initial-import-8cdbc214e8596521.yaml
new file mode 100644
index 0000000..85d3372
--- /dev/null
+++ b/releasenotes/notes/initial-import-8cdbc214e8596521.yaml
@@ -0,0 +1,24 @@
+---
+prelude: >
+ This is the first public release of the OpenStack Event Listener (OSEL).
+ It had previously been a project within Comcast, but was open-sourced
+ under the Apache license.
+features:
+ - |
+ Connects to RabbitMQ to listen for notification events specific to security
+ group changes. When those are intercepted, query Nova for information about
+ what the affected IP addresses are, then initiate a Qualys scan. Finally
+ send info in the IP addresses and the Qualys scan ID to syslog.
+issues:
+ - |
+ Only processes security group changes, should also process new port events
+ as well.
+ - |
+ Needs to exponential backoff for AMQP connections.
+ - |
+ Needs to be integrated with Aodh for modern OpenStacks.
+security:
+ - |
+ Requires access to RabbitMQ as well as OpenStack credentials that have access
+ to data in all projects, so this should be considered a privileged process and
+ should be run in a properly secured context.
diff --git a/security_group_events.go b/security_group_events.go
new file mode 100644
index 0000000..c9d158f
--- /dev/null
+++ b/security_group_events.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "strings"
+)
+
+// EventSecurityGroupRuleChange is the event processor class for all changes to
+// security groups. This includes additions and deletions. This must conform
+// to the EventProcessor interface (see events.go).
+type EventSecurityGroupRuleChange struct {
+ ChangeType string
+}
+
+// FillExtraData takes a security group change and enriches it with additional
+// information about the affected IP addresses using the
+// OpenStackActionInterface getPortList function.
+func (s EventSecurityGroupRuleChange) FillExtraData(e *Event, openstack OpenStackActioner) error {
+ // PopulateIps: This function returns a map of security group to array of IP addresses for all ports in the specified tenantID.
+
+ err := openstack.Connect(e.EventData.TenantID, e.EventData.UserName)
+ if err != nil {
+ return err
+ }
+
+ // Make port list request to neutron
+ resultMap := openstack.GetPortList()
+ resultIPAddresses := make(map[string][]string)
+ for _, ipMap := range resultMap {
+ resultIPAddresses[ipMap.securityGroup] = append(resultIPAddresses[ipMap.securityGroup], ipMap.ipAddress)
+ }
+ e.IPs = resultIPAddresses
+ return nil
+}
+
+// FormatLogs takes the accumulated event data and composes the JSON message to
+// be logged.
+func (s EventSecurityGroupRuleChange) FormatLogs(e *Event, scannedIPAddresses []string) ([]string, error) {
+ var es osSecurityGroupRuleChange
+ var logLines []string
+ if e == nil {
+ return logLines, fmt.Errorf("Event must not be nil")
+ }
+
+ if err := json.Unmarshal(e.RawData, &es); err != nil {
+ return logLines, err
+ }
+
+ hostName, err := os.Hostname()
+ if err != nil {
+ return nil, err
+ }
+ es.Payload.ChangeType = s.ChangeType
+ es.Payload.SourceType = OselVersion
+ es.Payload.SourceMessageBus = hostName
+ es.Payload.QualysScanID = e.QualysScanID
+ es.Payload.QualysScanError = e.QualysScanError
+
+ affectedIPArray := e.IPs[es.Payload.SecurityGroupRule.SecurityGroupID]
+ qualysScanJoin := fmt.Sprintf("|%s|", strings.Join(scannedIPAddresses, "|"))
+ for _, affectedIPAddr := range affectedIPArray {
+ es.Payload.QualysScanID = ""
+ es.Payload.QualysScanError = ""
+ if strings.Index(qualysScanJoin, fmt.Sprintf("|%s|", affectedIPAddr)) > -1 {
+ es.Payload.QualysScanID = e.QualysScanID
+ es.Payload.QualysScanError = e.QualysScanError
+ } else {
+ es.Payload.QualysScanID = ""
+ es.Payload.QualysScanError = "Not scanned by Qualys"
+ }
+ es.Payload.AffectedIPAddr = affectedIPAddr
+ jsonLine, err := json.Marshal(es.Payload)
+ if err != nil {
+ return nil, err
+ }
+ logLines = append(logLines, string(jsonLine))
+ }
+ return logLines, nil
+}
diff --git a/structs.go b/structs.go
new file mode 100644
index 0000000..3d2fded
--- /dev/null
+++ b/structs.go
@@ -0,0 +1,80 @@
+package main
+
+type openStackEvent struct {
+ EventType string `json:"event_type"`
+ Timestamp string `json:"timestamp"`
+ TenantID string `json:"_context_tenant_id"`
+ TenantName string `json:"_context_tenant_name"`
+ User string `json:"_context_user"`
+ UserName string `json:"_context_user_name"`
+ UserID string `json:"_context_user_id"`
+ IsAdmin bool `json:"_context_is_admin"`
+ PublisherID string `json:"publisher_id"`
+ MessageID string `json:"message_id"`
+}
+
+type osSecurityGroupRule struct {
+ RemoteGroupID interface{} `json:"remote_group_id"`
+ Direction string `json:"direction"`
+ Protocol interface{} `json:"protocol"`
+ RemoteIPPrefix string `json:"remote_ip_prefix"`
+ PortRangeMax interface{} `json:"port_range_max"`
+ // Dscp interface{} `json:"dscp"`
+ Rule string `json:"rule_direction"`
+ SecurityGroupID string `json:"security_group_id"`
+ TenantID string `json:"tenant_id"`
+ PortRangeMin interface{} `json:"port_range_min"`
+ Ethertype string `json:"ethertype"`
+ ID string `json:"id"`
+}
+
+type osSecurityGroupRuleChange struct {
+ Payload struct {
+ AffectedIPAddr interface{} `json:"affected_ip_address"`
+ ChangeType string `json:"change_type"`
+ QualysScanID string `json:"qualys_scan_id"`
+ QualysScanError string `json:"qualys_scan_error"`
+ SecurityGroupRule osSecurityGroupRule `json:"security_group_rule"`
+ SourceType string `json:"source_type"`
+ SourceMessageBus string `json:"source_message_bus"`
+ } `json:"payload"`
+}
+
+type osSecurityGroupRuleDelete struct {
+ Payload struct {
+ SecurityGroupRuleID string `json:"security_group_rule_id"`
+ } `json:"payload"`
+}
+
+type osPortCreate struct {
+ Payload struct {
+ Port osPort `json:"port"`
+ } `json:"payload"`
+}
+
+type osPort struct {
+ Status string `json:"status"`
+ BindingHostID string `json:"binding:host_id"`
+ Name string `json:"name"`
+ AllowedAddressPairs []interface{} `json:"allowed_address_pairs"`
+ AdminStateUp bool `json:"admin_state_up"`
+ NetworkID string `json:"network_id"`
+ TenantID string `json:"tenant_id"`
+ BindingVifDetails struct {
+ PortFilter bool `json:"port_filter"`
+ OvsHybridPlug bool `json:"ovs_hybrid_plug"`
+ } `json:"binding:vif_details"`
+ BindingVnicType string `json:"binding:vnic_type"`
+ BindingVifType string `json:"binding:vif_type"`
+ DeviceOwner string `json:"device_owner"`
+ MacAddress string `json:"mac_address"`
+ BindingProfile struct {
+ } `json:"binding:profile"`
+ FixedIps []struct {
+ SubnetID string `json:"subnet_id"`
+ IPAddress string `json:"ip_address"`
+ } `json:"fixed_ips"`
+ ID string `json:"id"`
+ SecurityGroups []string `json:"security_groups"`
+ DeviceID string `json:"device_id"`
+}
diff --git a/syslog.go b/syslog.go
new file mode 100644
index 0000000..e948c09
--- /dev/null
+++ b/syslog.go
@@ -0,0 +1,81 @@
+package main
+
+/*
+
+syslog - This file includes all of the logic necessary to interact with syslog.
+This is extrapolated out so that a SyslogActioner interface can be
+passed to functions. Doing this allows testing by mock classes to be created
+that can be passed to functions.
+
+Since this is a wrapper around the log/syslog library, this does not need
+testing.
+
+*/
+
+import (
+ "fmt"
+ "log"
+ "log/syslog"
+ "net"
+)
+
+// SyslogActioner is an interface for an SyslogActions class. Having
+// this as an interface allows us to pass in a dummy class for testing that
+// just returns mocked data.
+type SyslogActioner interface {
+ Connect() error
+ Info(string)
+}
+
+// SyslogActions is a class that handles all interactions directly with Syslog.
+// See the comment on SyslogActioner for rationale.
+type SyslogActions struct {
+ logger *syslog.Writer
+ Options SyslogOptions
+}
+
+// SyslogOptions is a class to convey all of the configurable options for the
+// SyslogActions class.
+type SyslogOptions struct {
+ Host string
+ Protocol string
+ Retry bool
+ Port string
+}
+
+// Info is the main method for the SyslogActioner class, it writes an
+// info-level message to the syslog stream.
+func (s SyslogActions) Info(writeMe string) {
+ log.Println("Logged: ", writeMe)
+ s.logger.Info(writeMe)
+}
+
+// Connect is the method that establishes the connection to the syslog server
+// over the network.
+func (s *SyslogActions) Connect() error {
+ var err error
+
+ address := net.JoinHostPort(s.Options.Host, s.Options.Port)
+
+ if Debug {
+ log.Printf("Opening %q syslog socket to %q\n", s.Options.Protocol, s.Options.Host)
+ }
+
+ s.logger, err = syslog.Dial(s.Options.Protocol, address, syslog.LOG_INFO, "osel")
+ if err != nil {
+ log.Printf("error opening syslog socket to %s: %s\n", s.Options.Host, err)
+ if s.Options.Retry {
+ for err != nil {
+ log.Println("retrying")
+ s.logger, err = syslog.Dial(s.Options.Protocol, address, syslog.LOG_INFO, "osel")
+ }
+ }
+ return fmt.Errorf("error opening syslog socket to %s: %s", s.Options.Host, err)
+ }
+
+ if Debug {
+ log.Println("Successfully connected to syslog host", s.Options.Host)
+ }
+
+ return nil
+}
diff --git a/syslog_mock_test.go b/syslog_mock_test.go
new file mode 100644
index 0000000..ba93c25
--- /dev/null
+++ b/syslog_mock_test.go
@@ -0,0 +1,24 @@
+package main
+
+import "log"
+
+type SyslogTestActions struct {
+ savedLogs []string
+}
+
+func (s *SyslogTestActions) Connect() error {
+ return nil
+}
+
+func (s *SyslogTestActions) Info(writeMe string) {
+ log.Printf("FAKE SYSLOG LINE: %s\n", writeMe)
+ s.savedLogs = append(s.savedLogs, writeMe)
+}
+
+func (s *SyslogTestActions) GetLogs() []string {
+ return s.savedLogs
+}
+
+func connectFakeSyslog() *SyslogTestActions {
+ return new(SyslogTestActions)
+}
diff --git a/tools/test-setup.sh b/tools/test-setup.sh
new file mode 100755
index 0000000..8d8b148
--- /dev/null
+++ b/tools/test-setup.sh
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# You cannot go build-time dependency fetching from projects hosted on github
+# without a github token, otherwise you get restricted by API throttling.
+# See: https://github.com/golang/go/issues/23955
+echo "machine api.github.com login openstackzuul password dba1634cb701f1c514f3268784b1d0a6512c12d4" >> $HOME/.netrc
+mkdir -p /home/zuul/go/src/v 2>/dev/null
+
+# Setup the environment prior to testing.
+export PATH=$PATH:$GOPATH/bin
+
+# Get OS
+case $(uname -s) in
+ Darwin)
+ OS=darwin
+ ;;
+ Linux)
+ if LSB_RELEASE=$(which lsb_release); then
+ OS=$($LSB_RELEASE -s -c)
+ else
+ # No lsb-release, trya hack or two
+ if which dpkg 1>/dev/null; then
+ OS=debian
+ elif which yum 1>/dev/null || which dnf 1>/dev/null; then
+ OS=redhat
+ else
+ echo "Linux distro not yet supported"
+ exit 1
+ fi
+ fi
+ ;;
+ *)
+ echo "Unsupported OS"
+ exit 1
+ ;;
+esac
+echo "Depected OS is '$OS'"
+
+echo | sudo -S /bin/true 2>/dev/null
+if [ $? != 0 ]; then
+ echo "Sudo does not work, so packages can not be installed"
+ exit 1
+fi
+
+# Now install go
+case $OS in
+ xenial)
+ sudo add-apt-repository ppa:longsleep/golang-backports
+ sudo apt-get update
+ sudo apt-get install -y golang-go golint
+ ;;
+esac
+
+# Install vgo https://github.com/golang/go/wiki/vgo
+if which go 1>/dev/null; then
+ sudo go get -u -v golang.org/x/vgo
+else
+ echo "go not found, install golang from source?"
+fi
diff --git a/viper.go b/viper.go
new file mode 100644
index 0000000..4d64c6f
--- /dev/null
+++ b/viper.go
@@ -0,0 +1,55 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/nate-johnston/viper"
+)
+
+// ViperConfig holds information per config item. If an Default
+// is not set then it is assumed that the value is Required.
+// NOTE: You can not set a default on a nested value. i.e. a value
+// within a has in a json or yaml file. (nested.value) you can
+// set nested values as required.
+type ViperConfig struct {
+ Key string // The config key that is required.
+ Default interface{} // Default Value to set.
+ Alias []string // Any key Aliases that should be registered
+ Description string // Description of the config.
+}
+
+// InitViper with the passed path and config.
+func InitViper(path string, viperConfigs []ViperConfig) error {
+ viper.SetConfigFile(path)
+ if err := viper.ReadInConfig(); err != nil {
+ return err
+ }
+ if err := ValidateConfig(viperConfigs); err != nil {
+ return err
+ }
+ return nil
+}
+
+// ValidateConfig will check the defined var ViperConfigs []ViperConfig and validate
+// the existances of the required keys, and set defaults for all keys where defaults are
+// defined.
+func ValidateConfig(viperConfigs []ViperConfig) error {
+ var errs []error
+ for _, rc := range viperConfigs {
+ if rc.Default == nil && viper.Get(rc.Key) == nil {
+ errs = append(errs, fmt.Errorf("Key: %s, Description: %s", rc.Key, rc.Description))
+ } else {
+ viper.SetDefault(rc.Key, rc.Default)
+ }
+
+ if len(rc.Alias) > 0 {
+ for _, a := range rc.Alias {
+ viper.RegisterAlias(a, rc.Key)
+ }
+ }
+ }
+ if len(errs) > 0 {
+ return fmt.Errorf("Required Configuration Missing: %v", errs)
+ }
+ return nil
+}
diff --git a/viper_test.go b/viper_test.go
new file mode 100644
index 0000000..6a339ed
--- /dev/null
+++ b/viper_test.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+ "github.com/nate-johnston/viper"
+ "github.com/stretchr/testify/assert"
+ "testing"
+)
+
+func TestInitViperWillReturnErrorOnReadError(t *testing.T) {
+ err := InitViper("fixtures/viper/not_there.yml", nil)
+ assert.NotNil(t, err)
+ assert.Equal(t, "open fixtures/viper/not_there.yml: no such file or directory", err.Error())
+}
+
+func TestInitViperEnsureItCallsValidateConfigWithConfigErrors(t *testing.T) {
+ // By calling InitViper and expecting an error case we prove that IniViper is
+ // calling ValidateConfig() and its working as expected.
+ viperConfigs := []ViperConfig{
+ ViperConfig{Key: "missing_required_string", Description: "Required String"},
+ }
+ err := InitViper("fixtures/viper/test.yml", viperConfigs)
+ assert.NotNil(t, err)
+ assert.Equal(t, "Required Configuration Missing: [Key: missing_required_string, Description: Required String]",
+ err.Error())
+}
+
+func TestValidateConfig(t *testing.T) {
+ viperConfigs := []ViperConfig{
+ ViperConfig{Key: "required_string", Description: "Required String"},
+ ViperConfig{Key: "nested.one", Description: "Required Nested One"},
+ ViperConfig{Key: "test_default", Default: "Optional Value", Description: "Optional Value"},
+ ViperConfig{Key: "test_alias", Alias: []string{"bubba", "forest"}, Description: "Optional Value"},
+ }
+ InitViper("fixtures/viper/test.yml", viperConfigs)
+
+ err := ValidateConfig(viperConfigs)
+ assert.Nil(t, err)
+ assert.Equal(t, "Optional Value", viper.GetString("test_default"))
+ assert.Equal(t, "test_alias value", viper.GetString("test_alias"))
+ assert.Equal(t, "test_alias value", viper.GetString("bubba"))
+ assert.Equal(t, "test_alias value", viper.GetString("forest"))
+}