T864 - Separate log file for unparsed queries

acra-censor/acra-censor_configuration_provider.go

@@ -19,6 +19,7 @@ package acracensor
 import (
 	"github.com/cossacklabs/acra/acra-censor/handlers"
 	"gopkg.in/yaml.v2"
+	"os"
 	"strings"
 )
 
@@ -39,7 +40,8 @@ type Config struct {
 		Patterns []string
 		Filepath string
 	}
-	IgnoreParseError bool `yaml:"ignore_parse_error"`
+	IgnoreParseError bool   `yaml:"ignore_parse_error"`
+	ParseErrorsLog   string `yaml:"parse_errors_log"`
 }
 
 // LoadConfiguration loads configuration of AcraCensor
@@ -50,6 +52,15 @@ func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) error {
 		return err
 	}
 	acraCensor.ignoreParseError = censorConfiguration.IgnoreParseError
+	acraCensor.parseErrorsLogPath = censorConfiguration.ParseErrorsLog
+	if acraCensor.parseErrorsLogPath != "" {
+		openedFile, err := os.OpenFile(acraCensor.parseErrorsLogPath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			return err
+		}
+		defer openedFile.Close()
+	}
+
 	for _, handlerConfiguration := range censorConfiguration.Handlers {
 		switch handlerConfiguration.Handler {
 		case WhitelistConfigStr:

acra-censor/acra-censor_implementation.go

@@ -20,23 +20,26 @@ import (
 	"github.com/cossacklabs/acra/acra-censor/common"
 	"github.com/cossacklabs/acra/acra-censor/handlers"
 	log "github.com/sirupsen/logrus"
+	"os"
 )
 
 // ServiceName to use in logs
 const ServiceName = "acra-censor"
 
 // AcraCensor describes censor data: query handler, logger and reaction on parsing errors.
 type AcraCensor struct {
-	handlers         []QueryHandlerInterface
-	ignoreParseError bool
-	logger           *log.Entry
+	handlers           []QueryHandlerInterface
+	ignoreParseError   bool
+	parseErrorsLogPath string
+	logger             *log.Entry
 }
 
 // NewAcraCensor creates new censor object.
 func NewAcraCensor() *AcraCensor {
 	acraCensor := &AcraCensor{}
 	acraCensor.logger = log.WithField("service", ServiceName)
 	acraCensor.ignoreParseError = false
+	acraCensor.parseErrorsLogPath = ""
 	return acraCensor
 }
 
@@ -72,6 +75,13 @@ func (acraCensor *AcraCensor) HandleQuery(rawQuery string) error {
 	normalizedQuery, queryWithHiddenValues, parsedQuery, err := common.HandleRawSQLQuery(rawQuery)
 	if err == common.ErrQuerySyntaxError {
 		acraCensor.logger.WithError(err).Warning("Failed to parse input query")
+		if acraCensor.parseErrorsLogPath != "" {
+			err := acraCensor.saveQuery(rawQuery)
+			if err != nil {
+				acraCensor.logger.WithError(err).Errorf("An error occurred while saving unparsable query")
+				return err
+			}
+		}
 		if acraCensor.ignoreParseError {
 			acraCensor.logger.Infof("Unparsed query has been allowed")
 			return nil
@@ -110,3 +120,19 @@ func (acraCensor *AcraCensor) HandleQuery(rawQuery string) error {
 	acraCensor.logger.Infof("Allowed query: '%s'", queryWithHiddenValues)
 	return nil
 }
+
+func (acraCensor *AcraCensor) saveQuery(rawQuery string) error {
+	openedFile, err := os.OpenFile(acraCensor.parseErrorsLogPath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0600)
+	if err != nil {
+		return err
+	}
+
+	defer openedFile.Close()
+
+	_, err = openedFile.WriteString(rawQuery + "\n")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

acra-censor/acra-censor_test.go

@@ -23,6 +23,7 @@ limitations under the License.
 package acracensor
 
 import (
+	"bufio"
 	"github.com/cossacklabs/acra/acra-censor/common"
 	"io/ioutil"
 	"os"
@@ -1734,6 +1735,10 @@ func TestConfigurationProvider(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		err = os.Remove("unparsed_queries_log")
+		if err != nil {
+			t.Fatal(err)
+		}
 	}()
 	err = acraCensor.LoadConfiguration(configuration)
 	if err != nil {
@@ -1771,13 +1776,14 @@ func TestConfigurationProvider(t *testing.T) {
 		"SELECT EMP_ID, LAST_NAME FROM EMPLOYEE WHERE CITY = 'Seattle' ORDER BY EMP_ID;",
 		//"SELECT EMP_ID, LAST_NAME FROM EMPLOYEE AS EMPL WHERE CITY = 'Seattle' ORDER BY EMP_ID;",
 	}
-	//acracensor should block those structures
+	//acracensor should block those queries by pattern
 	for _, queryToBlock := range testQueries {
 		err = acraCensor.HandleQuery(queryToBlock)
 		if err != common.ErrBlacklistPatternMatch {
 			t.Fatal(err)
 		}
 	}
+
 	for _, currentHandler := range acraCensor.handlers {
 		original, ok := currentHandler.(*handlers.QueryCaptureHandler)
 		if ok {
@@ -1894,3 +1900,74 @@ func TestIgnoringQueryParseErrors(t *testing.T) {
 	// check when censor with two handlers and each one will return query parse error
 	checkHandler([]QueryHandlerInterface{whitelist, blacklist}, nil)
 }
+func TestLogUnparsedQueries(t *testing.T) {
+	var err error
+
+	configuration := `ignore_parse_error: true
+parse_errors_log: unparsed_queries_log
+handlers:
+  - handler: query_capture
+    filepath: censor_log
+  - handler: query_ignore
+    queries:
+      - ROLLBACK
+      - COMMIT
+      - BEGIN
+  - handler: blacklist
+    queries:
+      - INSERT INTO SalesStaff1 VALUES (1, 'Stephen', 'Jiang');
+      - SELECT AVG(Price) FROM Products;
+    tables:
+      - EMPLOYEE_TBL
+      - Customers
+    patterns:
+      - SELECT EMP_ID, LAST_NAME FROM EMPLOYEE %%WHERE%%;`
+
+	acraCensor := NewAcraCensor()
+	defer func() {
+		acraCensor.ReleaseAll()
+		err = os.Remove("censor_log")
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = os.Remove("unparsed_queries_log")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	err = acraCensor.LoadConfiguration([]byte(configuration))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testQueries := []string{
+		"select * from x )))(((unparsable query",
+		"qwerty",
+	}
+
+	for _, query := range testQueries {
+		err = acraCensor.HandleQuery(query)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	file, err := os.Open("unparsed_queries_log")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	i := 0
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		if scanner.Text() != testQueries[i] {
+			t.Fatal("Scanned: " + scanner.Text() + ", expected: " + testQueries[i])
+		}
+		i++
+	}
+
+	if err := scanner.Err(); err != nil {
+		t.Fatal(err)
+	}
+}

configs/acra-censor.example.yaml

@@ -1,4 +1,5 @@
 ignore_parse_error: false
+parse_errors_log: unparsed_queries_log
 handlers:
   - handler: query_capture
     filepath: censor_log