T864 - Separate log file for unparsed queries

acra-censor/acra-censor_configuration_provider.go

@@ -17,17 +17,17 @@ limitations under the License.
 package acracensor
 
 import (
+	"github.com/cossacklabs/acra/acra-censor/common"
 	"github.com/cossacklabs/acra/acra-censor/handlers"
 	"gopkg.in/yaml.v2"
 	"strings"
 )
 
 // Query handlers' names.
 const (
-	BlacklistConfigStr    = "blacklist"
-	WhitelistConfigStr    = "whitelist"
-	QueryCaptureConfigStr = "query_capture"
-	QueryIgnoreConfigStr  = "query_ignore"
+	BlacklistConfigStr   = "blacklist"
+	WhitelistConfigStr   = "whitelist"
+	QueryIgnoreConfigStr = "query_ignore"
 )
 
 // Config shows handlers configuration: queries, tables, patterns
@@ -37,9 +37,10 @@ type Config struct {
 		Queries  []string
 		Tables   []string
 		Patterns []string
-		Filepath string
 	}
-	IgnoreParseError bool `yaml:"ignore_parse_error"`
+	IgnoreParseError bool   `yaml:"ignore_parse_error"`
+	ParseErrorsLog   string `yaml:"parse_errors_log"`
+	CaptureLog       string `yaml:"capture_log"`
 }
 
 // LoadConfiguration loads configuration of AcraCensor
@@ -50,6 +51,21 @@ func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) error {
 		return err
 	}
 	acraCensor.ignoreParseError = censorConfiguration.IgnoreParseError
+	if !strings.EqualFold(censorConfiguration.CaptureLog, "") {
+		queryWriter, err := common.NewQueryWriter(censorConfiguration.CaptureLog)
+		if err != nil {
+			return err
+		}
+		acraCensor.parsedQueriesWriter = queryWriter
+	}
+	if !strings.EqualFold(censorConfiguration.ParseErrorsLog, "") {
+		queryWriter, err := common.NewQueryWriter(censorConfiguration.ParseErrorsLog)
+		if err != nil {
+			return err
+		}
+		acraCensor.unparsedQueriesWriter = queryWriter
+	}
+
 	for _, handlerConfiguration := range censorConfiguration.Handlers {
 		switch handlerConfiguration.Handler {
 		case WhitelistConfigStr:
@@ -78,16 +94,6 @@ func (acraCensor *AcraCensor) LoadConfiguration(configuration []byte) error {
 			}
 			acraCensor.AddHandler(blacklistHandler)
 			break
-		case QueryCaptureConfigStr:
-			if strings.EqualFold(handlerConfiguration.Filepath, "") {
-				break
-			}
-			queryCaptureHandler, err := handlers.NewQueryCaptureHandler(handlerConfiguration.Filepath)
-			if err != nil {
-				return err
-			}
-			acraCensor.AddHandler(queryCaptureHandler)
-			break
 		case QueryIgnoreConfigStr:
 			queryIgnoreHandler := handlers.NewQueryIgnoreHandler()
 			queryIgnoreHandler.AddQueries(handlerConfiguration.Queries)

acra-censor/acra-censor_implementation.go

@@ -18,25 +18,30 @@ package acracensor
 
 import (
 	"github.com/cossacklabs/acra/acra-censor/common"
-	"github.com/cossacklabs/acra/acra-censor/handlers"
 	log "github.com/sirupsen/logrus"
+	"strings"
+	"time"
 )
 
 // ServiceName to use in logs
 const ServiceName = "acra-censor"
 
 // AcraCensor describes censor data: query handler, logger and reaction on parsing errors.
 type AcraCensor struct {
-	handlers         []QueryHandlerInterface
-	ignoreParseError bool
-	logger           *log.Entry
+	handlers              []QueryHandlerInterface
+	ignoreParseError      bool
+	parsedQueriesWriter   *common.QueryWriter
+	unparsedQueriesWriter *common.QueryWriter
+	logger                *log.Entry
 }
 
 // NewAcraCensor creates new censor object.
 func NewAcraCensor() *AcraCensor {
 	acraCensor := &AcraCensor{}
 	acraCensor.logger = log.WithField("service", ServiceName)
 	acraCensor.ignoreParseError = false
+	acraCensor.parsedQueriesWriter = nil
+	acraCensor.unparsedQueriesWriter = nil
 	return acraCensor
 }
 
@@ -65,37 +70,25 @@ func (acraCensor *AcraCensor) ReleaseAll() {
 
 // HandleQuery processes every query through each handler.
 func (acraCensor *AcraCensor) HandleQuery(rawQuery string) error {
-	if len(acraCensor.handlers) == 0 {
+	if len(acraCensor.handlers) == 0 && acraCensor.parsedQueriesWriter == nil && acraCensor.unparsedQueriesWriter == nil {
 		// no handlers, AcraCensor won't work
 		return nil
 	}
 	normalizedQuery, queryWithHiddenValues, parsedQuery, err := common.HandleRawSQLQuery(rawQuery)
+	// Unparsed query handling
 	if err == common.ErrQuerySyntaxError {
 		acraCensor.logger.WithError(err).Warning("Failed to parse input query")
+		acraCensor.saveUnparsedQuery(rawQuery)
 		if acraCensor.ignoreParseError {
-			acraCensor.logger.Infof("Unparsed query has been allowed")
+			acraCensor.logger.Infoln("Unparsed query has been allowed")
 			return nil
 		}
-		acraCensor.logger.Errorf("Unparsed query has been forbidden")
+		acraCensor.logger.Errorln("Unparsed query has been forbidden")
 		return err
 	}
-
+	// Parsed query handling
+	acraCensor.saveParsedQuery(queryWithHiddenValues)
 	for _, handler := range acraCensor.handlers {
-		// in QueryCapture Handler we use only redacted queries
-		if queryCaptureHandler, ok := handler.(*handlers.QueryCaptureHandler); ok {
-			queryCaptureHandler.CheckQuery(queryWithHiddenValues, parsedQuery)
-			continue
-		}
-		// in QueryIgnore Handler we use only raw queries
-		if queryIgnoreHandler, ok := handler.(*handlers.QueryIgnoreHandler); ok {
-			continueHandling, _ := queryIgnoreHandler.CheckQuery(rawQuery, parsedQuery)
-			if continueHandling {
-				continue
-			} else {
-				break
-			}
-		}
-		// remained handlers operate
 		continueHandling, err := handler.CheckQuery(normalizedQuery, parsedQuery)
 		if err != nil {
 			acraCensor.logger.Errorf("Forbidden query: '%s'", queryWithHiddenValues)
@@ -110,3 +103,40 @@ func (acraCensor *AcraCensor) HandleQuery(rawQuery string) error {
 	acraCensor.logger.Infof("Allowed query: '%s'", queryWithHiddenValues)
 	return nil
 }
+
+// GetLoggingTimeout returns current timeout of censor's logging process
+func (acraCensor *AcraCensor) GetLoggingTimeout() time.Duration {
+	return acraCensor.parsedQueriesWriter.GetSerializationTimeout()
+}
+
+// SetLoggingTimeout sets timeout of censor's logging process
+func (acraCensor *AcraCensor) SetLoggingTimeout(duration time.Duration) {
+	acraCensor.parsedQueriesWriter.SetSerializationTimeout(duration)
+	acraCensor.unparsedQueriesWriter.SetSerializationTimeout(duration)
+}
+
+func (acraCensor *AcraCensor) saveUnparsedQuery(query string) {
+	if acraCensor.unparsedQueriesWriter != nil {
+		saveQuery(acraCensor.unparsedQueriesWriter, query)
+	}
+}
+
+func (acraCensor *AcraCensor) saveParsedQuery(query string) {
+	if acraCensor.parsedQueriesWriter != nil {
+		saveQuery(acraCensor.parsedQueriesWriter, query)
+	}
+}
+
+func saveQuery(writer *common.QueryWriter, query string) {
+	//skip already captured queries
+	for _, capturedQuery := range writer.Queries {
+		if strings.EqualFold(capturedQuery.RawQuery, query) {
+			return
+		}
+	}
+	queryInfo := &common.QueryInfo{}
+	queryInfo.RawQuery = query
+	queryInfo.IsForbidden = false
+	writer.Queries = append(writer.Queries, queryInfo)
+	writer.BufferedQueries = append(writer.BufferedQueries, queryInfo)
+}

acra-censor/acra-censor_interfaces.go

@@ -22,7 +22,10 @@ limitations under the License.
 // https://github.com/cossacklabs/acra/wiki/AcraCensor
 package acracensor
 
-import "github.com/cossacklabs/acra/sqlparser"
+import (
+	"github.com/cossacklabs/acra/sqlparser"
+	"time"
+)
 
 // QueryHandlerInterface describes what actions are available for queries.
 type QueryHandlerInterface interface {
@@ -36,4 +39,6 @@ type AcraCensorInterface interface {
 	AddHandler(handler QueryHandlerInterface)
 	RemoveHandler(handler QueryHandlerInterface)
 	ReleaseAll()
+	GetLoggingTimeout() time.Duration
+	SetLoggingTimeout(duration time.Duration)
 }