Showing preview only (2,163K chars total). Download the full file or copy to clipboard to get everything.
Repository: nielsbasjes/logparser
Branch: main
Commit: 0e5ab16f6270
Files: 212
Total size: 2.0 MB
Directory structure:
gitextract_a39vjp3u/
├── .editorconfig
├── .github/
│ ├── FUNDING.yml
│ └── workflows/
│ └── build.yml
├── .gitignore
├── .pmd
├── .travis.yml__
├── CHANGELOG.md
├── CNAME
├── GeoIP2-TestData/
│ ├── Dockerfile
│ ├── README.md
│ ├── rebuild.sh
│ ├── source-data/
│ │ ├── GeoIP2-City-Test.json
│ │ ├── GeoIP2-Country-Test.json
│ │ ├── GeoIP2-ISP-Test.json
│ │ └── GeoLite2-ASN-Test.json
│ └── test-data/
│ ├── GeoIP2-City-Test.mmdb
│ ├── GeoIP2-Country-Test.mmdb
│ ├── GeoIP2-ISP-Test.mmdb
│ ├── GeoLite2-ASN-Test.mmdb
│ └── write-test-data.pl
├── LICENSE
├── README-Hive.md
├── README-Java.md
├── README-Pig.md
├── README-geoip.md
├── README.md
├── _config.yml
├── devtools/
│ ├── docker/
│ │ ├── Dockerfile
│ │ ├── bashcolors.sh
│ │ ├── build_env_checks.sh
│ │ ├── configure-for-user.sh
│ │ ├── env.sh
│ │ └── prompt.sh
│ ├── logformat.conf
│ ├── pom.xml
│ ├── release.sh
│ └── src/
│ └── main/
│ └── resources/
│ └── checkstyle/
│ ├── checkstyle.xml
│ └── suppressions.xml
├── docs/
│ ├── CNAME
│ └── README.md
├── examples/
│ ├── apache-beam/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── avro/
│ │ │ └── Record.avdl
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── beam/
│ │ │ ├── TestCase.java
│ │ │ ├── avro/
│ │ │ │ ├── ExpectedClick.java
│ │ │ │ ├── TestParserDoFnAvro.java
│ │ │ │ └── TestParserDoFnAvroInline.java
│ │ │ └── pojo/
│ │ │ ├── MyRecord.java
│ │ │ ├── TestParserDoFnClass.java
│ │ │ └── TestParserDoFnInline.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── apache-flink/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── avro/
│ │ │ └── Record.avdl
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── flink/
│ │ │ ├── TestCase.java
│ │ │ ├── avro/
│ │ │ │ ├── ExpectedClick.java
│ │ │ │ ├── TestParserMapFunctionAvroClass.java
│ │ │ │ └── TestParserMapFunctionAvroInline.java
│ │ │ └── pojo/
│ │ │ ├── MyRecord.java
│ │ │ ├── TestParserMapFunctionClass.java
│ │ │ └── TestParserMapFunctionInline.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── apache-hadoop-mapreduce/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── main/
│ │ ├── assembly/
│ │ │ └── job.xml
│ │ └── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── hadoop/
│ │ └── io/
│ │ └── input/
│ │ └── Wordcount.java
│ ├── demolog/
│ │ ├── README.md
│ │ └── hackers-access.log
│ ├── java-pojo/
│ │ ├── pom.xml
│ │ └── src/
│ │ └── main/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ ├── Main.java
│ │ │ └── MyRecord.java
│ │ └── resources/
│ │ └── log4j.properties
│ └── pom.xml
├── httpdlog/
│ ├── httpdlog-inputformat/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── assembly/
│ │ │ │ └── job.xml
│ │ │ └── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── hadoop/
│ │ │ └── input/
│ │ │ ├── ApacheHttpdLogfileInputFormat.java
│ │ │ ├── ApacheHttpdLogfileRecordReader.java
│ │ │ └── ParsedRecord.java
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── hadoop/
│ │ │ └── input/
│ │ │ ├── TestApacheHttpdLogfileInputFormat.java
│ │ │ ├── TestGetAllFields.java
│ │ │ └── TestParsedRecord.java
│ │ └── resources/
│ │ ├── access.log
│ │ └── log4j.properties
│ ├── httpdlog-parser/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── antlr4/
│ │ │ │ └── nl/
│ │ │ │ └── basjes/
│ │ │ │ └── parse/
│ │ │ │ └── strftime/
│ │ │ │ └── StrfTime.g4
│ │ │ ├── assembly/
│ │ │ │ └── job.xml
│ │ │ ├── java/
│ │ │ │ └── nl/
│ │ │ │ └── basjes/
│ │ │ │ └── parse/
│ │ │ │ └── httpdlog/
│ │ │ │ ├── ApacheHttpdLogFormatDissector.java
│ │ │ │ ├── HttpdLogFormatDissector.java
│ │ │ │ ├── HttpdLoglineParser.java
│ │ │ │ ├── NginxHttpdLogFormatDissector.java
│ │ │ │ ├── Utils.java
│ │ │ │ └── dissectors/
│ │ │ │ ├── HttpFirstLineDissector.java
│ │ │ │ ├── HttpFirstLineProtocolDissector.java
│ │ │ │ ├── HttpUriDissector.java
│ │ │ │ ├── ModUniqueIdDissector.java
│ │ │ │ ├── QueryStringFieldDissector.java
│ │ │ │ ├── RequestCookieListDissector.java
│ │ │ │ ├── ResponseSetCookieDissector.java
│ │ │ │ ├── ResponseSetCookieListDissector.java
│ │ │ │ ├── ScreenResolutionDissector.java
│ │ │ │ ├── StrfTimeStampDissector.java
│ │ │ │ ├── StrfTimeToDateTimeFormatter.java
│ │ │ │ ├── TimeStampDissector.java
│ │ │ │ ├── geoip/
│ │ │ │ │ ├── AbstractGeoIPDissector.java
│ │ │ │ │ ├── GeoIPASNDissector.java
│ │ │ │ │ ├── GeoIPCityDissector.java
│ │ │ │ │ ├── GeoIPCountryDissector.java
│ │ │ │ │ └── GeoIPISPDissector.java
│ │ │ │ ├── nginxmodules/
│ │ │ │ │ ├── CoreLogModule.java
│ │ │ │ │ ├── GeoIPModule.java
│ │ │ │ │ ├── KubernetesIngressModule.java
│ │ │ │ │ ├── NginxModule.java
│ │ │ │ │ ├── SslModule.java
│ │ │ │ │ ├── UpstreamListDissector.java
│ │ │ │ │ ├── UpstreamModule.java
│ │ │ │ │ └── VariousModule.java
│ │ │ │ ├── tokenformat/
│ │ │ │ │ ├── NamedTokenParser.java
│ │ │ │ │ ├── ParameterizedTokenParser.java
│ │ │ │ │ ├── Token.java
│ │ │ │ │ ├── TokenFormatDissector.java
│ │ │ │ │ ├── TokenOutputField.java
│ │ │ │ │ ├── TokenParser.java
│ │ │ │ │ └── TokenSorterByStartPos.java
│ │ │ │ └── translate/
│ │ │ │ ├── ConvertCLFIntoNumber.java
│ │ │ │ ├── ConvertMillisecondsIntoMicroseconds.java
│ │ │ │ ├── ConvertNumberIntoCLF.java
│ │ │ │ ├── ConvertSecondsWithMillisStringDissector.java
│ │ │ │ └── TypeConvertBaseDissector.java
│ │ │ └── resources/
│ │ │ └── version/
│ │ │ └── Version.java.template
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ ├── ApacheHttpdAllFieldsTest.java
│ │ │ ├── ApacheHttpdLogParserTest.java
│ │ │ ├── BasicOverallTest.java
│ │ │ ├── ClientHintsTest.java
│ │ │ ├── CookiesTest.java
│ │ │ ├── EdgeCasesTest.java
│ │ │ ├── JettyLogFormatParserTest.java
│ │ │ ├── JsonLogFormatTest.java
│ │ │ ├── MultiLineHttpdLogParserTest.java
│ │ │ ├── NginxLogFormatJsonTest.java
│ │ │ ├── NginxLogFormatTest.java
│ │ │ ├── UtilsTest.java
│ │ │ ├── dissectors/
│ │ │ │ ├── TestCookieDissector.java
│ │ │ │ ├── TestGeoIPDissectors.java
│ │ │ │ ├── TestHttpFirstLineDissector.java
│ │ │ │ ├── TestHttpUriDissector.java
│ │ │ │ ├── TestModUniqueIdDissector.java
│ │ │ │ ├── TestQueryStringDissector.java
│ │ │ │ └── TestTimeStampDissector.java
│ │ │ ├── nginxmodules/
│ │ │ │ ├── NginxAllFieldsTest.java
│ │ │ │ └── NginxUpstreamTest.java
│ │ │ └── translate/
│ │ │ └── TestTranslators.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── httpdlog-serde/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── assembly/
│ │ │ │ └── udf.xml
│ │ │ └── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── ApacheHttpdlogDeserializer.java
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ ├── TestAllDissectorTypes.java
│ │ │ └── TestApacheHttpdlogDeserializer.java
│ │ └── resources/
│ │ └── log4j.properties
│ └── pom.xml
├── parser-core/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── job.xml
│ │ └── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── parse/
│ │ └── core/
│ │ ├── Casts.java
│ │ ├── Dissector.java
│ │ ├── Field.java
│ │ ├── Parsable.java
│ │ ├── ParsedField.java
│ │ ├── Parser.java
│ │ ├── SimpleDissector.java
│ │ ├── Value.java
│ │ └── exceptions/
│ │ ├── DissectionFailure.java
│ │ ├── FatalErrorDuringCallOfSetterMethod.java
│ │ ├── InvalidDissectorException.java
│ │ ├── InvalidFieldMethodSignature.java
│ │ └── MissingDissectorsException.java
│ └── test/
│ ├── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── parse/
│ │ └── core/
│ │ ├── ParserCastsTest.java
│ │ ├── ParserDissectionOutputTypesTest.java
│ │ ├── ParserDuplicateOutputTest.java
│ │ ├── ParserExceptionsTest.java
│ │ ├── ParserInfiniteLoopTest.java
│ │ ├── ParserNormalTest.java
│ │ ├── ParserNormalTestRecord.java
│ │ ├── ParserResetTest.java
│ │ ├── ParserTypeColissionTest.java
│ │ ├── ParserTypeRemappingEdgeCase.java
│ │ ├── TestBadAPIUsage.java
│ │ ├── annotation/
│ │ │ ├── TestFieldSetters.java
│ │ │ ├── TestFieldSettersAlwaysCombined.java
│ │ │ ├── TestFieldSettersAlwaysSeparate.java
│ │ │ ├── TestFieldSettersNotEmpty.java
│ │ │ └── TestFieldSettersNotNull.java
│ │ ├── convert/
│ │ │ └── ValueConvertTest.java
│ │ ├── reference/
│ │ │ ├── BarDissector.java
│ │ │ ├── FooDissector.java
│ │ │ ├── FooSpecialDissector.java
│ │ │ ├── ReferenceTest.java
│ │ │ └── ReferenceTestDouble.java
│ │ └── test/
│ │ ├── DissectorTester.java
│ │ ├── EmptyValuesDissector.java
│ │ ├── MyDissectorTester.java
│ │ ├── NormalValuesDissector.java
│ │ ├── NullValuesDissector.java
│ │ ├── TestRecord.java
│ │ ├── TestUltimateDummyDissector.java
│ │ ├── TestUltimateDummyDissectorFailurelogging.java
│ │ └── UltimateDummyDissector.java
│ └── resources/
│ └── log4j.properties
├── pom.xml
├── renovate.json
├── start-docker.sh
└── utils/
├── PojoGenerator/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── nl/
│ └── basjes/
│ └── parse/
│ └── httpdlog/
│ └── PojoGenerator.java
└── pom.xml
================================================
FILE CONTENTS
================================================
================================================
FILE: .editorconfig
================================================
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
root = true
[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
[*.java]
indent_size = 4
[*.xml]
indent_size = 2
# Tab indentation (no size specified)
================================================
FILE: .github/FUNDING.yml
================================================
github: nielsbasjes
custom: https://www.paypal.me/nielsbasjes
================================================
FILE: .github/workflows/build.yml
================================================
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: Logparser
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
name: Build with Java ${{ matrix.java }}
runs-on: ubuntu-latest
steps:
- name: Checkout sourcecode
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Cache Local Maven Repository
uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: 'Setup: Install JDK 21, 25'
uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
with:
distribution: 'temurin'
java-version: |
21
25
- name: Build Logparser
run: mvn -B clean package
- name: Codecov
uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
with:
token: ${{ secrets.CODECOV_TOKEN }}
================================================
FILE: .gitignore
================================================
*.class
# Maven
dependency-reduced-pom.xml
# Package Files #
*.jar
*.war
*.ear
target
*.bak
*.swp
*~
bin
__*
#Eclipse stuff
.metadata
.settings
.classpath
.project
hs_err_pid*.log
#IntelliJ stuff
.idea
*.iml
#test files
x
xx
y
yy
z
zz
#Hive/Hadoop logs
SecurityAuth-*.audit
hadoop*.log
*-audit.log
rm-appsummary.log
# Docker
devtools/docker/_m2/
devtools/docker/_gnupg/
================================================
FILE: .pmd
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!--
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<pmd>
<useProjectRuleSet>false</useProjectRuleSet>
<ruleSetFile>.ruleset</ruleSetFile>
<rules>
<rule>
<name>AvoidDecimalLiteralsInBigDecimalConstructor</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>AvoidMultipleUnaryOperators</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>AvoidThreadGroup</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>AvoidUsingHardCodedIP</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>AvoidUsingOctalValues</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>BigIntegerInstantiation</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>BooleanInstantiation</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>BrokenNullCheck</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>CheckResultSet</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>ClassCastExceptionWithToArray</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>CollapsibleIfStatements</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>DoubleCheckedLocking</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyCatchBlock</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyFinallyBlock</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyIfStmt</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyInitializer</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyStatementNotInLoop</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyStaticInitializer</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptySwitchStatements</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptySynchronizedBlock</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyTryBlock</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>EmptyWhileStmt</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>ForLoopShouldBeWhileLoop</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>JumbledIncrementer</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>MisplacedNullCheck</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>OverrideBothEqualsAndHashcode</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>ReturnFromFinallyBlock</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UnconditionalIfStatement</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryConversionTemporary</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryFinalModifier</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryReturn</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UnusedNullCheckInEquals</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UselessOperationOnImmutable</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>UselessOverridingMethod</name>
<ruleset>Basic Rules</ruleset>
</rule>
<rule>
<name>ForLoopsMustUseBraces</name>
<ruleset>Braces Rules</ruleset>
</rule>
<rule>
<name>IfElseStmtsMustUseBraces</name>
<ruleset>Braces Rules</ruleset>
</rule>
<rule>
<name>IfStmtsMustUseBraces</name>
<ruleset>Braces Rules</ruleset>
</rule>
<rule>
<name>WhileLoopsMustUseBraces</name>
<ruleset>Braces Rules</ruleset>
</rule>
<rule>
<name>CloneThrowsCloneNotSupportedException</name>
<ruleset>Clone Implementation Rules</ruleset>
</rule>
<rule>
<name>ProperCloneImplementation</name>
<ruleset>Clone Implementation Rules</ruleset>
</rule>
<rule>
<name>CyclomaticComplexity</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>ExcessiveClassLength</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>ExcessiveMethodLength</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>ExcessiveParameterList</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>ExcessivePublicCount</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>NcssConstructorCount</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>NcssMethodCount</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>NcssTypeCount</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>NPathComplexity</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>TooManyFields</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>TooManyMethods</name>
<ruleset>Code Size Rules</ruleset>
</rule>
<rule>
<name>AssignmentInOperand</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>AtLeastOneConstructor</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>AvoidAccessibilityAlteration</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>AvoidUsingNativeCode</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>AvoidUsingShortType</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>AvoidUsingVolatile</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>BooleanInversion</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>CallSuperInConstructor</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>DefaultPackage</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>DoNotCallGarbageCollectionExplicitly</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>DontImportSun</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>SuspiciousOctalEscape</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryConstructor</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryParentheses</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>UnusedModifier</name>
<ruleset>Controversial Rules</ruleset>
</rule>
<rule>
<name>CouplingBetweenObjects</name>
<ruleset>Coupling Rules</ruleset>
</rule>
<rule>
<name>ExcessiveImports</name>
<ruleset>Coupling Rules</ruleset>
</rule>
<rule>
<name>AbstractClassWithoutAbstractMethod</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AbstractClassWithoutAnyMethod</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AccessorClassGeneration</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AssignmentToNonFinalStatic</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidConstantsInterface</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidDeeplyNestedIfStmts</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidInstanceofChecksInCatchClause</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidProtectedFieldInFinalClass</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidReassigningParameters</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidSynchronizedAtMethodLevel</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>BadComparison</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>ClassWithOnlyPrivateConstructorsShouldBeFinal</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>CloseResource</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>CompareObjectsWithEquals</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>ConfusingTernary</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>ConstructorCallsOverridableMethod</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>DefaultLabelNotLastInSwitchStmt</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>EmptyMethodInAbstractClassShouldBeAbstract</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>EqualsNull</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>FinalFieldCouldBeStatic</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>IdempotentOperations</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>ImmutableField</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>InstantiationToGetClass</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>MissingBreakInSwitch</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>MissingStaticMethodInNonInstantiatableClass</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>NonCaseLabelInSwitchStatement</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>NonStaticInitializer</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>NonThreadSafeSingleton</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>OptimizableToArrayCall</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>PositionLiteralsFirstInComparisons</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>PreserveStackTrace</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>ReturnEmptyArrayRatherThanNull</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SimpleDateFormatNeedsLocale</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SimplifyBooleanExpressions</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SimplifyBooleanReturns</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SimplifyConditional</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SingularField</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SwitchDensity</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>SwitchStmtsShouldHaveDefault</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>TooFewBranchesForASwitchStatement</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UncommentedEmptyConstructor</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UncommentedEmptyMethod</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryLocalBeforeReturn</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UnsynchronizedStaticDateFormatter</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UseCollectionIsEmpty</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UseLocaleWithCaseConversions</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UseNotifyAllInsteadOfNotify</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>UseSingleton</name>
<ruleset>Design Rules</ruleset>
</rule>
<rule>
<name>AvoidCallingFinalize</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>EmptyFinalizer</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>FinalizeDoesNotCallSuperFinalize</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>FinalizeOnlyCallsSuperFinalize</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>FinalizeOverloaded</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>FinalizeShouldBeProtected</name>
<ruleset>Finalizer Rules</ruleset>
</rule>
<rule>
<name>DontImportJavaLang</name>
<ruleset>Import Statement Rules</ruleset>
</rule>
<rule>
<name>DuplicateImports</name>
<ruleset>Import Statement Rules</ruleset>
</rule>
<rule>
<name>ImportFromSamePackage</name>
<ruleset>Import Statement Rules</ruleset>
</rule>
<rule>
<name>TooManyStaticImports</name>
<ruleset>Import Statement Rules</ruleset>
</rule>
<rule>
<name>DoNotCallSystemExit</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>DoNotUseThreads</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>LocalHomeNamingConvention</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>LocalInterfaceSessionNamingConvention</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>MDBAndSessionBeanNamingConvention</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>RemoteInterfaceNamingConvention</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>RemoteSessionInterfaceNamingConvention</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>StaticEJBFieldShouldBeFinal</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>UseProperClassLoader</name>
<ruleset>J2EE Rules</ruleset>
</rule>
<rule>
<name>ProperLogger</name>
<ruleset>Jakarta Commons Logging Rules</ruleset>
</rule>
<rule>
<name>UseCorrectExceptionLogging</name>
<ruleset>Jakarta Commons Logging Rules</ruleset>
</rule>
<rule>
<name>AvoidPrintStackTrace</name>
<ruleset>Java Logging Rules</ruleset>
</rule>
<rule>
<name>LoggerIsNotStaticFinal</name>
<ruleset>Java Logging Rules</ruleset>
</rule>
<rule>
<name>MoreThanOneLogger</name>
<ruleset>Java Logging Rules</ruleset>
</rule>
<rule>
<name>SystemPrintln</name>
<ruleset>Java Logging Rules</ruleset>
</rule>
<rule>
<name>MissingSerialVersionUID</name>
<ruleset>JavaBean Rules</ruleset>
</rule>
<rule>
<name>JUnitAssertionsShouldIncludeMessage</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>JUnitSpelling</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>JUnitStaticSuite</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>JUnitTestsShouldIncludeAssert</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>SimplifyBooleanAssertion</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>TestClassWithoutTestCases</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryBooleanAssertion</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>UseAssertEqualsInsteadOfAssertTrue</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>UseAssertNullInsteadOfAssertTrue</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>UseAssertSameInsteadOfAssertTrue</name>
<ruleset>JUnit Rules</ruleset>
</rule>
<rule>
<name>AvoidAssertAsIdentifier</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>AvoidEnumAsIdentifier</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>ByteInstantiation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>IntegerInstantiation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>JUnit4SuitesShouldUseSuiteAnnotation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>JUnit4TestShouldUseAfterAnnotation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>JUnit4TestShouldUseBeforeAnnotation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>JUnit4TestShouldUseTestAnnotation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>JUnitUseExpected</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>LongInstantiation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>ReplaceEnumerationWithIterator</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>ReplaceHashtableWithMap</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>ReplaceVectorWithList</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>ShortInstantiation</name>
<ruleset>Migration Rules</ruleset>
</rule>
<rule>
<name>AbstractNaming</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>AvoidDollarSigns</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>AvoidFieldNameMatchingMethodName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>AvoidFieldNameMatchingTypeName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>BooleanGetMethodName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>ClassNamingConventions</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>LongVariable</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>MethodNamingConventions</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>MethodWithSameNameAsEnclosingClass</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>MisleadingVariableName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>NoPackage</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>PackageCase</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>ShortMethodName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>ShortVariable</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>SuspiciousConstantFieldName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>SuspiciousEqualsMethodName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>SuspiciousHashcodeMethodName</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>VariableNamingConventions</name>
<ruleset>Naming Rules</ruleset>
</rule>
<rule>
<name>AddEmptyString</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>AvoidArrayLoops</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>AvoidInstantiatingObjectsInLoops</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>MethodArgumentCouldBeFinal</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>SimplifyStartsWith</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryWrapperObjectCreation</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>UseArrayListInsteadOfVector</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>UseArraysAsList</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>UseStringBufferForStringAppends</name>
<ruleset>Optimization Rules</ruleset>
</rule>
<rule>
<name>ArrayIsStoredDirectly</name>
<ruleset>Security Code Guidelines</ruleset>
</rule>
<rule>
<name>MethodReturnsInternalArray</name>
<ruleset>Security Code Guidelines</ruleset>
</rule>
<rule>
<name>AvoidCatchingNPE</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AvoidCatchingThrowable</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AvoidRethrowingException</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AvoidThrowingNewInstanceOfSameException</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AvoidThrowingNullPointerException</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AvoidThrowingRawExceptionTypes</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>DoNotExtendJavaLangError</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>DoNotThrowExceptionInFinally</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>ExceptionAsFlowControl</name>
<ruleset>Strict Exception Rules</ruleset>
</rule>
<rule>
<name>AppendCharacterWithChar</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>AvoidDuplicateLiterals</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>AvoidStringBufferField</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>ConsecutiveLiteralAppends</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>InefficientEmptyStringCheck</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>InefficientStringBuffering</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>InsufficientStringBufferDeclaration</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>StringBufferInstantiationWithChar</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>StringInstantiation</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>StringToString</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>UnnecessaryCaseChange</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>UseEqualsToCompareStrings</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>UseIndexOfChar</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>UselessStringValueOf</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>UseStringBufferLength</name>
<ruleset>String and StringBuffer Rules</ruleset>
</rule>
<rule>
<name>CloneMethodMustImplementCloneable</name>
<ruleset>Type Resolution Rules</ruleset>
</rule>
<rule>
<name>LooseCoupling</name>
<ruleset>Type Resolution Rules</ruleset>
</rule>
<rule>
<name>SignatureDeclareThrowsException</name>
<ruleset>Type Resolution Rules</ruleset>
</rule>
<rule>
<name>UnusedImports</name>
<ruleset>Type Resolution Rules</ruleset>
</rule>
<rule>
<name>UnusedFormalParameter</name>
<ruleset>Unused Code Rules</ruleset>
</rule>
<rule>
<name>UnusedLocalVariable</name>
<ruleset>Unused Code Rules</ruleset>
</rule>
<rule>
<name>UnusedPrivateField</name>
<ruleset>Unused Code Rules</ruleset>
</rule>
<rule>
<name>UnusedPrivateMethod</name>
<ruleset>Unused Code Rules</ruleset>
</rule>
</rules>
<includeDerivedFiles>false</includeDerivedFiles>
<violationsAsErrors>true</violationsAsErrors>
</pmd>
================================================
FILE: .travis.yml__
================================================
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
language: java
dist: trusty
jdk:
- oraclejdk8
after_success:
- mvn -PEnableReportPlugins coveralls:report
cache:
directories:
- '$HOME/.m2/repository'
================================================
FILE: CHANGELOG.md
================================================
This is intended as an overview of the major changes
v6.0.1-SNAPSHOT
===
- ...
v6.0.0
===
- Updated to need Java 21!
- Update Serde to Hive 4.2.0
- Updated Flink and Beam examples to latest versions
- GeoIP update no longer supports the "metrocode"
v5.11.0
===
- Update Serde to Hive 4.0.0
v5.10.0
===
- Do not fail on logline specific parse errors, simply not provide the expected dissections.
v5.9.0
===
- Print the parsed values of a wildcard in DissectorTester::printAllPossibleValues
- Allow bad requests to be parsed anyway.
- Drop Apache Pig support (this tool is no longer used by anyone)
v5.8
===
- getPossiblePaths sorts the list by fieldname.
- Removed GeoIP fields averageincome and populationdensity which are not part of any real mmdb file.
- Dropped the already disabled Storm example
- Fully switched to Junit 5
- Require JDK 11 or newer to build
- Workaround for change in Unicode CLDR(and thus Java 17): they changed the short name of "September" in Locale.UK to "Sept" which causes parse errors.
- Fixed bug regarding escaped characters in headers.
v5.7
===
- Updated dependencies
- When adding a type remapping with an explicit cast this cast was lost and replaced by "STRING_ONLY".
v5.6
===
- Fix bug that in some cases values would be reported multiple times.
v5.5
===
- Handle HTML encoded values in the URL better
v5.4
===
- Updated many dependencies
- Fixed extracting the timezone
- In url parsing missing values are now 'absent' (i.e. not set)
- Handle loglines when the upstream module is not running (i.e. fields are a '-')
v5.3
===
- Updated many dependencies
v5.2
===
- Improve regex performance
- Added basic support for the NGinx Upstream, SSL, GeoIP and all other documented modules
- Added basic support for parsing the Kubernetes Ingress logformat variables
- Disallow some Dissector methods to return a null.
- Updated many dependencies: Hadoop, Flink, Beam etc.
v5.1
===
- Parse epoch seconds `%{%s}t`
- Added GeoIP2 dissectors for City, Country and ASN data.
- Improved output of dissector testing framework.
v5.0
===
- The %u specifier allowed a space which broke parsing if the field after it also allowed space.
- If a custom time format does not contain a timezone we assume a default timezone (UTC).
- Fix extracting milliseconds, added extracting microseconds and nanoseconds.
- Allow `%{%usec_frac}t` in addition to `%{usec_frac}t` (same msec_frac).
- Introduce a setterPolicy to determine if the setter is called in case of a NULL or EMPTY value.
- Replace Cobertura with Jacoco because of Java 8
- Remove Yauaa from tests and examples because of circular dependency between the projects.
- Make Java API more fluent (breaks backwards compatibility with external dissectors).
v4.0
===
- Switching to require Java 8
- Parser instance is now serializable.
- Added example on using with Apache Flink
- Added example on using with Apache Beam
- Rewrote Apache Storm code and move to examples
- Many changes in Exception handling
v3.1
===
- Handle illegal data: Double # in the URL
- Handle illegal data: Firstline is rubbish (Reported by Yong Zhang - java8964).
v3.0
===
- Accept the NGinx logformat.
- Output the httpd parser git info, build timestamp and version info on startup.
- Fixed problem when using a different root dissector than provided by default.
- Created a testing toolkit for dissectors and improved several tests.
- Allow a dissector to add an additional dissector (needed for custom time format parsing)
- Support for parsing many (not all) of the possible custom time format fields.
- Allow changing the parser even after first use (makes it more flexible to use).
- Allow a dissector to return an empty 'extra part' to allow an alternate form of type remapping
- Improved test coverage
- Dissection the setcookies expire value was not deterministic and an absent 'expire' value is now returned as null instead of the 'now' timestamp.
- Implemented `%{UNIT}T`
- Implemented converters between several closely related formats (BYTES/BYTESCLF, time related formats)
- Token based parsers can output multiple values for the same parameter.
- Implemented all < and > directives for Apache logformat
- Implemented `%{VARNAME}^ti` and `%{VARNAME}^to`
**RELEVANT CHANGES COMPARED WITH THE 2.X VERSION**:
These fields are now reported as deprecated.
- `%b` changed from "BYTES:response.body.bytesclf" to "BYTESCLF:response.body.bytes"
- `%D` changed from "server.process.time" to "response.server.processing.time"
- `%{msec_frac}t` changed from "request.receive.time.begin.msec_frac" to "request.receive.time.msec_frac"
- `%{usec_frac}t` changed from "request.receive.time.begin.usec_frac" to "request.receive.time.usec_frac"
- `%{msec}t` changed from "request.receive.time.begin.msec" to "request.receive.time.msec"
- `%{usec}t` changed from "request.receive.time.begin.usec" to "request.receive.time.usec"
v2.8
===
- Allow parsing mixed case timeformats
- Added the ISO 8601 'date' output for parsed times (Is string "yyyy-MM-dd")
- Added the ISO 8601 'time' output for parsed times (Is string "HH:mm:ss" )
- Solve parse error when a HTTP method like "VERSION-CONTROL" is used.
- Fixed NPE in specific combination of cast and setter type and a null value
- Improved test coverage
v2.7
===
- Handle the effects of mod_reqtimeout giving a http 408 (reported by Diogo Sant'Ana)
- Added option to optionally continue even when some requested dissectors are missing (Java only).
v2.6
===
- Buildin fix for the problems in Jetty logging
v2.5
===
- Simply treat the `%{...}t` as a text field you can retrieve (instead of failing if it occurs).
- Change URI dissector to allow URIs like android-app://...
- Change URI dissector to allow % in the URI when it is not an escape sequence (like in `?promo=Give-5%-discount`)
v2.4
===
- Rewrote the way parsed values are passed around. Improves accuracy and performance in specific cases.
- Now support parsing the first line even if it is chopped by Apache httpd because of an URI longer than 8000 bytes.
- Fixed an infinite recursion problem.
- Fixed Timestamp unit test (test was broken, code was fine).
v2.3
===
- The raw timestamp extracted from the Apache logfiles no longer contains the surrounding '[' ']'.
v2.2
===
- Accept multiple logformat lines as a single 'multiline' input string.
v2.1.1
===
- Fixed simple problem in the PIG example output
v2.1
===
- Dissect the unique ID from mod_unique_id.
- [PIG] Make getting the example code for PIG a bit easier
v2.0
===
- Fixed reading logfiles from before 2000
- Rearranged the Java packages to make the structure more logical.
- Changed license from GPL to Apache v2.0
v1.9.1
===
- Allow urls that have a space (which is incorrect but does happen).
v1.9
===
- [PIG] Support for getting a map[] of values (useful for cookies and query string parameters)
- [PIG] Output the possible values as a complete working Pig statement.
Older
===
Just see the commit logs
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: CNAME
================================================
logparser.basjes.nl
================================================
FILE: GeoIP2-TestData/Dockerfile
================================================
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM perl:5.40.0-buster
RUN cpan File::Slurper
RUN cpan Cpanel::JSON::XS
RUN cpan Math::Int128
RUN cpan MaxMind::DB::Writer::Serializer
WORKDIR "/GeoIPTestData/test-data"
CMD ["/GeoIPTestData/test-data/write-test-data.pl"]
================================================
FILE: GeoIP2-TestData/README.md
================================================
I copied some of the files from https://github.com/maxmind/MaxMind-DB/
and modified them to suit my own needs.
These base files are only used for unit tests in this project and are licensed as
This work is licensed under the Creative Commons Attribution-ShareAlike 3.0
Unported License. To view a copy of this license, visit
http://creativecommons.org/licenses/by-sa/3.0/ or send a letter to Creative
Commons, 444 Castro Street, Suite 900, Mountain View, California, 94041, USA.
================================================
FILE: GeoIP2-TestData/rebuild.sh
================================================
#!/bin/bash
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
cd "${DIR}" || exit 1
PROJECTNAME=GeoIPTestData
IMAGE_NAME=geo-ip-testdatabuilder
CONTAINER_NAME=GeoIPTestDataBuilder
docker build -t "${IMAGE_NAME}" .
if [ "$(uname -s)" == "Linux" ]; then
USER_NAME=${SUDO_USER:=${USER}}
USER_ID=$(id -u "${USER_NAME}")
GROUP_ID=$(id -g "${USER_NAME}")
else # boot2docker uid and gid
USER_NAME=${USER}
USER_ID=1000
GROUP_ID=50
fi
# man docker-run
# When using SELinux, mounted directories may not be accessible
# to the container. To work around this, with Docker prior to 1.7
# one needs to run the "chcon -Rt svirt_sandbox_file_t" command on
# the directories. With Docker 1.7 and later the z mount option
# does this automatically.
# Since Docker 1.7 was release 5 years ago we only support 1.7 and newer.
V_OPTS=:z
COMMAND=( "$@" )
if [ $# -eq 0 ];
then
# COMMAND=( "bash" "-i" )
COMMAND=( )
fi
( cd "${DIR}/test-data" && rm *.mmdb )
docker run --rm=true -i -t \
-u "${USER_ID}" \
-v "${PWD}:/GeoIPTestData${V_OPTS:-}" \
-w "/GeoIPTestData/test-data" \
--name "${CONTAINER_NAME}" \
"${IMAGE_NAME}" \
"${COMMAND[@]}"
================================================
FILE: GeoIP2-TestData/source-data/GeoIP2-City-Test.json
================================================
[
{
"::80.100.47.45/118": {
"city": {
"confidence": 1,
"geoname_id": "1234",
"names": {
"en": "Amstelveen"
}
},
"postal": {
"code": "1187",
"confidence": 2
},
"subdivisions": [
{
"confidence": 3,
"geoname_id": 2345,
"iso_code": "NH",
"names": {
"en": "Noord Holland"
}
}
],
"continent": {
"code": "EU",
"geoname_id": 6255148,
"names": {
"de": "Europa",
"en": "Europe",
"es": "Europa",
"fr": "Europe",
"ja": "ヨーロッパ",
"pt-BR": "Europa",
"ru": "Европа",
"zh-CN": "欧洲"
}
},
"country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"confidence": 42,
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
},
"location": {
"accuracy_radius": 4,
"latitude": "52.5",
"longitude": "5.75",
"time_zone": "Europe/Amsterdam",
"metro_code" : "5"
},
"registered_country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
}
}
},
{
"2001:980:91c0::/56": {
"city": {
"confidence": 11,
"geoname_id": "1234",
"names": {
"en": "Amstelveen"
}
},
"postal": {
"code": "1187",
"confidence": 12
},
"subdivisions": [
{
"confidence": 13,
"geoname_id": 2345,
"iso_code": "NH",
"names": {
"en": "Noord Holland"
}
}
],
"continent": {
"code": "EU",
"geoname_id": 6255148,
"names": {
"de": "Europa",
"en": "Europe",
"es": "Europa",
"fr": "Europe",
"ja": "ヨーロッパ",
"pt-BR": "Europa",
"ru": "Европа",
"zh-CN": "欧洲"
}
},
"country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"confidence": 42,
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
},
"location": {
"accuracy_radius": 14,
"latitude": "52.5",
"longitude": "5.75",
"time_zone": "Europe/Amsterdam",
"metro_code": "15"
},
"registered_country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
}
}
}
]
================================================
FILE: GeoIP2-TestData/source-data/GeoIP2-Country-Test.json
================================================
[
{
"::80.100.47.45/118": {
"continent": {
"code": "EU",
"geoname_id": 6255148,
"names": {
"de": "Europa",
"en": "Europe",
"es": "Europa",
"fr": "Europe",
"ja": "ヨーロッパ",
"pt-BR": "Europa",
"ru": "Европа",
"zh-CN": "欧洲"
}
},
"country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"confidence": 42,
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
},
"registered_country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
}
}
},
{
"2001:980:91c0::/56": {
"continent": {
"code": "EU",
"geoname_id": 6255148,
"names": {
"de": "Europa",
"en": "Europe",
"es": "Europa",
"fr": "Europe",
"ja": "ヨーロッパ",
"pt-BR": "Europa",
"ru": "Европа",
"zh-CN": "欧洲"
}
},
"country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"confidence": 42,
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
},
"registered_country": {
"geoname_id": 2750405,
"is_in_european_union": true,
"iso_code": "NL",
"names": {
"de": "Niederlande",
"en": "Netherlands",
"es": "Holanda",
"fr": "Pays-Bas",
"ja": "オランダ王国",
"pt-BR": "Países Baixos",
"ru": "Нидерланды",
"zh-CN": "荷兰"
}
}
}
}
]
================================================
FILE: GeoIP2-TestData/source-data/GeoIP2-ISP-Test.json
================================================
[
{
"::80.100.47.45/118": {
"autonomous_system_number": 4444,
"autonomous_system_organization": "Basjes Global Network",
"isp": "Basjes ISP",
"organization": "Niels Basjes"
}
},
{
"2001:980:91c0::/56": {
"autonomous_system_number": 6666,
"autonomous_system_organization": "Basjes Global Network IPv6",
"isp": "Basjes ISP IPv6",
"organization": "Niels Basjes IPv6"
}
}
]
================================================
FILE: GeoIP2-TestData/source-data/GeoLite2-ASN-Test.json
================================================
[
{
"::80.100.47.45/118": {
"autonomous_system_number": 4444,
"autonomous_system_organization": "Basjes Global Network"
}
},
{
"2001:980:91c0::/56": {
"autonomous_system_number": 6666,
"autonomous_system_organization": "Basjes Global Network IPv6"
}
}
]
================================================
FILE: GeoIP2-TestData/test-data/write-test-data.pl
================================================
#!/usr/bin/env perl
use strict;
use warnings;
use autodie;
use utf8;
use Cwd qw( abs_path );
use File::Basename qw( dirname );
use File::Slurper qw( read_binary write_binary );
use Cpanel::JSON::XS 4.16 qw( decode_json );
use Math::Int128 qw( MAX_UINT128 string_to_uint128 uint128 );
use MaxMind::DB::Writer::Serializer 0.100004;
use MaxMind::DB::Writer::Tree 0.100004;
use MaxMind::DB::Writer::Util qw( key_for_data );
use Net::Works::Network ();
use Test::MaxMind::DB::Common::Util qw( standard_test_metadata );
my $Dir = dirname( abs_path($0) );
sub main {
# my @sizes = ( 24, 28, 32 );
# my @ipv4_range = ( '1.1.1.1', '1.1.1.32' );
# my @ipv4_subnets = Net::Works::Network->range_as_subnets(@ipv4_range);
# for my $record_size (@sizes) {
# write_test_db(
# $record_size,
# \@ipv4_subnets,
# { ip_version => 4 },
# 'ipv4',
# );
# }
# write_broken_pointers_test_db(
# 24,
# \@ipv4_subnets,
# { ip_version => 4 },
# 'broken-pointers',
# );
# write_broken_search_tree_db(
# 24,
# \@ipv4_subnets,
# { ip_version => 4 },
# 'broken-search-tree',
# );
# my @ipv6_subnets = Net::Works::Network->range_as_subnets(
# '::1:ffff:ffff',
# '::2:0000:0059'
# );
# for my $record_size (@sizes) {
# write_test_db(
# $record_size,
# \@ipv6_subnets,
# { ip_version => 6 },
# 'ipv6',
# );
#
# write_test_db(
# $record_size,
# [
# @ipv6_subnets,
# Net::Works::Network->range_as_subnets( @ipv4_range, 6 ),
# ],
# { ip_version => 6 },
# 'mixed',
# );
# }
# write_decoder_test_db();
# write_pointer_decoder_test_db();
# write_deeply_nested_structures_db();
write_geoip2_dbs();
# write_broken_geoip2_city_db();
# write_invalid_node_count();
# write_no_ipv4_tree_db();
# write_no_map_db( \@ipv4_subnets );
# write_test_serialization_data();
# write_db_with_metadata_pointers();
}
# sub write_broken_pointers_test_db {
# no warnings 'redefine';
#
# my $orig_store_data = MaxMind::DB::Writer::Serializer->can('store_data');
#
# # This breaks the value of the record for the 1.1.1.32 network, causing it
# # to point outside the database.
# local *MaxMind::DB::Writer::Serializer::store_data = sub {
# my $data_pointer = shift->$orig_store_data(@_);
# my $value = $_[1];
# if ( ref($value) eq 'HASH'
# && exists $value->{ip}
# && $value->{ip} eq '1.1.1.32' ) {
#
# $data_pointer += 100_000;
# }
# return $data_pointer;
# };
#
# # The next hack will poison the data section for the 1.1.16/28 subnet
# # value. It's value will be a pointer that resolves to an offset outside
# # the database.
#
# my $key_to_poison = key_for_data( { ip => '1.1.1.16' } );
#
# my $orig_position_for_data
# = MaxMind::DB::Writer::Serializer->can('_position_for_data');
# local *MaxMind::DB::Writer::Serializer::_position_for_data = sub {
# my $key = $_[1];
#
# if ( $key eq $key_to_poison ) {
# return 1_000_000;
# }
# else {
# return shift->$orig_position_for_data(@_);
# }
# };
#
# write_test_db(@_);
#
# return;
# }
# sub write_broken_search_tree_db {
# my $filename = ( write_test_db(@_) )[1];
#
# my $content = read_binary($filename);
#
# # This causes the right record of the first node to be 0, meaning it
# # points back to the top of the tree. This should never happen in a
# # database that follows the spec.
# substr( $content, 5, 1 ) = "\0";
# write_binary( $filename, $content );
#
# return;
# }
# sub write_test_db {
# my $record_size = shift;
# my $subnets = shift;
# my $metadata = shift;
# my $ip_version_name = shift;
#
# my $writer = MaxMind::DB::Writer::Tree->new(
# ip_version => $subnets->[0]->version(),
# record_size => $record_size,
# alias_ipv6_to_ipv4 => ( $subnets->[0]->version() == 6 ? 1 : 0 ),
# map_key_type_callback => sub { 'utf8_string' },
# standard_test_metadata(),
# %{$metadata},
# );
#
# for my $subnet ( @{$subnets} ) {
# $writer->insert_network(
# $subnet,
# { ip => $subnet->first()->as_string() }
# );
# }
#
# my $filename = sprintf(
# "$Dir/MaxMind-DB-test-%s-%i.mmdb",
# $ip_version_name,
# $record_size,
# );
# open my $fh, '>', $filename;
#
# $writer->write_tree($fh);
#
# close $fh;
#
# return ( $writer, $filename );
# }
# {
# # We will store this once for each subnet so we will also be testing
# # pointers, since the serializer will generate a pointer to this
# # structure.
# my %all_types = (
# utf8_string => 'unicode! ☯ - ♫',
# double => 42.123456,
# bytes => pack( 'N', 42 ),
# uint16 => 100,
# uint32 => 2**28,
# int32 => -1 * ( 2**28 ),
# uint64 => uint128(1) << 60,
# uint128 => uint128(1) << 120,
# array => [ 1, 2, 3, ],
# map => {
# mapX => {
# utf8_stringX => 'hello',
# arrayX => [ 7, 8, 9 ],
# },
# },
# boolean => 1,
# float => 1.1,
# );
#
# my %all_types_0 = (
# utf8_string => q{},
# double => 0,
# bytes => q{},
# uint16 => 0,
# uint32 => 0,
# int32 => 0,
# uint64 => uint128(0),
# uint128 => uint128(0),
# array => [],
# map => {},
# boolean => 0,
# float => 0,
# );
#
# # We limit this to numeric types as the other types would generate
# # very large databases
# my %numeric_types_max = (
# double => 'Inf',
# float => 'Inf',
# int32 => 0x7fffffff,
# uint16 => 0xffff,
# uint32 => string_to_uint128('0xffff_ffff'),
# uint64 => string_to_uint128('0xffff_ffff_ffff_ffff'),
# uint128 => MAX_UINT128,
# );
#
# sub write_decoder_test_db {
# my $writer = _decoder_writer();
#
# my @subnets
# = map { Net::Works::Network->new_from_string( string => $_ ) }
# qw(
# ::1.1.1.0/120
# ::2.2.0.0/112
# ::3.0.0.0/104
# ::4.5.6.7/128
# abcd::/64
# 1000::1234:0000/112
# );
#
# for my $subnet (@subnets) {
# $writer->insert_network(
# $subnet,
# \%all_types,
# );
# }
#
# $writer->insert_network(
# Net::Works::Network->new_from_string( string => '::0.0.0.0/128' ),
# \%all_types_0,
# );
#
# $writer->insert_network(
# Net::Works::Network->new_from_string(
# string => '::255.255.255.255/128'
# ),
# \%numeric_types_max,
# );
#
# open my $fh, '>', "$Dir/MaxMind-DB-test-decoder.mmdb";
# $writer->write_tree($fh);
# close $fh;
#
# return;
# }
#
# sub write_pointer_decoder_test_db {
#
# # We want to create a database where most values are pointers
# no warnings 'redefine';
# local *MaxMind::DB::Writer::Serializer::_should_cache_value
# = sub { 1 };
# my $writer = _decoder_writer();
#
# # We add these slightly different records so that we end up with
# # pointers for the individual values in the maps, not just pointers
# # to the map
# $writer->insert_network(
# '1.0.0.0/32',
# {
# %all_types,
# booleanX => 0,
# arrayX => [ 1, 2, 3, 4, ],
# mapXX => {
# utf8_stringX => 'hello',
# arrayX => [ 7, 8, 9, 10 ],
# booleanX => 0,
# },
# },
# );
#
# $writer->insert_network(
# '1.1.1.0/32',
# {
# %all_types,
#
# # This has to be 0 rather than 1 as otherwise the buggy
# # Perl writer will think it is the same as an uint32 value of
# # 1 and make a pointer to a value of a different type.
# boolean => 0,
# },
# );
#
# open my $fh, '>', "$Dir/MaxMind-DB-test-pointer-decoder.mmdb";
# $writer->write_tree($fh);
# close $fh;
#
# return;
# }
#
# sub _decoder_writer {
# return MaxMind::DB::Writer::Tree->new(
# ip_version => 6,
# record_size => 24,
# database_type => 'MaxMind DB Decoder Test',
# languages => ['en'],
# description => {
# en =>
# 'MaxMind DB Decoder Test database - contains every MaxMind DB data type',
# },
# alias_ipv6_to_ipv4 => 1,
# remove_reserved_networks => 0,
# map_key_type_callback => sub {
# my $key = $_[0];
# $key =~ s/X*$//;
# return $key eq 'array' ? [ 'array', 'uint32' ] : $key;
# },
# );
# }
# }
# {
# my %nested = (
# map1 => {
# map2 => {
# array => [
# {
# map3 => { a => 1, b => 2, c => 3 },
# },
# ],
# },
# },
# );
#
# sub write_deeply_nested_structures_db {
# my $writer = MaxMind::DB::Writer::Tree->new(
# ip_version => 6,
# record_size => 24,
# ip_version => 6,
# database_type => 'MaxMind DB Nested Data Structures',
# languages => ['en'],
# description => {
# en =>
# 'MaxMind DB Nested Data Structures Test database - contains deeply nested map/array structures',
# },
# alias_ipv6_to_ipv4 => 1,
# map_key_type_callback => sub {
# my $key = shift;
# return
# $key =~ /^map/ ? 'map'
# : $key eq 'array' ? [ 'array', 'map' ]
# : 'uint32';
# }
# );
#
# my @subnets
# = map { Net::Works::Network->new_from_string( string => $_ ) }
# qw(
# ::1.1.1.0/120
# ::2.2.0.0/112
# ::3.0.0.0/104
# ::4.5.6.7/128
# abcd::/64
# 1000::1234:0000/112
# );
#
# for my $subnet (@subnets) {
# $writer->insert_network(
# $subnet,
# \%nested,
# );
# }
#
# open my $fh, '>', "$Dir/MaxMind-DB-test-nested.mmdb";
# $writer->write_tree($fh);
# close $fh;
#
# return;
# }
# }
sub write_geoip2_dbs {
_write_geoip2_db( @{$_}[ 0, 1 ], 'Test' )
for (
# [ 'GeoIP2-Anonymous-IP', {} ],
['GeoIP2-City'],
# ['GeoIP2-Connection-Type'],
['GeoIP2-Country'],
# ['GeoIP2-DensityIncome'],
# ['GeoIP2-Domain'],
# ['GeoIP2-Enterprise'],
['GeoIP2-ISP'],
# ['GeoIP2-Precision-Enterprise'],
# ['GeoIP2-Static-IP-Score'],
# ['GeoIP2-User-Count'],
['GeoLite2-ASN'],
# ['GeoLite2-City'],
# ['GeoLite2-Country'],
);
}
#sub write_broken_geoip2_city_db {
# no warnings 'redefine';
#
# # This is how we _used_ to encode doubles. Storing them this way with the
# # current reader tools can lead to weird errors. This broken database is a
# # good way to test the robustness of reader code in the face of broken
# # databases.
# local *MaxMind::DB::Writer::Serializer::_encode_double = sub {
# my $self = shift;
# my $value = shift;
#
# $self->_simple_encode( double => $value );
# };
#
# _write_geoip2_db( 'GeoIP2-City', 0, 'Test Broken Double Format' );
#}
#sub write_invalid_node_count {
# no warnings 'redefine';
# local *MaxMind::DB::Writer::Tree::node_count = sub { 100000 };
#
# _write_geoip2_db( 'GeoIP2-City', 0, 'Test Invalid Node Count' );
#}
sub _universal_map_key_type_callback {
my $map = {
# languages
de => 'utf8_string',
en => 'utf8_string',
es => 'utf8_string',
fr => 'utf8_string',
ja => 'utf8_string',
'pt-BR' => 'utf8_string',
ru => 'utf8_string',
'zh-CN' => 'utf8_string',
# production
accuracy_radius => 'uint16',
autonomous_system_number => 'uint32',
autonomous_system_organization => 'utf8_string',
average_income => 'uint32',
city => 'map',
code => 'utf8_string',
confidence => 'uint16',
connection_type => 'utf8_string',
continent => 'map',
country => 'map',
domain => 'utf8_string',
geoname_id => 'uint32',
ipv4_24 => 'uint32',
ipv4_32 => 'uint32',
ipv6_32 => 'uint32',
ipv6_48 => 'uint32',
ipv6_64 => 'uint32',
is_anonymous => 'boolean',
is_anonymous_proxy => 'boolean',
is_anonymous_vpn => 'boolean',
is_hosting_provider => 'boolean',
is_in_european_union => 'boolean',
is_legitimate_proxy => 'boolean',
is_public_proxy => 'boolean',
is_residential_proxy => 'boolean',
is_satellite_provider => 'boolean',
is_tor_exit_node => 'boolean',
iso_code => 'utf8_string',
isp => 'utf8_string',
latitude => 'double',
location => 'map',
longitude => 'double',
metro_code => 'uint16',
names => 'map',
organization => 'utf8_string',
population_density => 'uint32', #FIXME: Was 'uint32' but that makes the Java code crash!
postal => 'map',
registered_country => 'map',
represented_country => 'map',
score => 'double',
static_ip_score => 'double',
subdivisions => [ 'array', 'map' ],
time_zone => 'utf8_string',
traits => 'map',
traits => 'map',
type => 'utf8_string',
user_type => 'utf8_string',
# for testing only
foo => 'utf8_string',
bar => 'utf8_string',
buzz => 'utf8_string',
our_value => 'utf8_string',
};
my $callback = sub {
my $key = shift;
return $map->{$key} || die <<"ERROR";
Unknown tree key '$key'.
The universal_map_key_type_callback doesn't know what type to use for the passed
key. If you are adding a new key that will be used in a frozen tree / mmdb then
you should update the mapping in both our internal code and here.
ERROR
};
return $callback;
}
sub _write_geoip2_db {
my $type = shift;
my $populate_all_networks_with_data = shift;
my $description = shift;
my $writer = MaxMind::DB::Writer::Tree->new(
ip_version => 6,
record_size => 28,
ip_version => 6,
database_type => $type,
languages => [ 'en', $type eq 'GeoIP2-City' ? ('zh') : () ],
description => {
en => ( $type =~ s/-/ /gr )
. " $description Database (fake GeoIP2 data, for example purposes only)",
$type eq 'GeoIP2-City' ? ( zh => '小型数据库' ) : (),
},
alias_ipv6_to_ipv4 => 1,
map_key_type_callback => _universal_map_key_type_callback(),
);
$writer->_set_build_epoch(1); # Fake timestamp to make the test files stable
_populate_all_networks( $writer, $populate_all_networks_with_data )
if $populate_all_networks_with_data;
my $value = shift;
my $nodes
= decode_json( read_binary("$Dir/../source-data/$type-Test.json") );
for my $node (@$nodes) {
for my $network ( keys %$node ) {
$writer->insert_network(
Net::Works::Network->new_from_string( string => $network ),
$node->{$network}
);
}
}
my $suffix = $description =~ s/ /-/gr;
open my $output_fh, '>', "$Dir/$type-$suffix.mmdb";
$writer->write_tree($output_fh);
close $output_fh;
return;
}
sub _populate_all_networks {
my $writer = shift;
my $data = shift;
my $max_uint128 = uint128(0) - 1;
my @networks = Net::Works::Network->range_as_subnets(
Net::Works::Address->new_from_integer(
integer => 0,
version => 6,
),
Net::Works::Address->new_from_integer(
integer => $max_uint128,
version => 6,
),
);
for my $network (@networks) {
$writer->insert_network( $network => $data );
}
}
# sub write_no_ipv4_tree_db {
# my $subnets = shift;
#
# my $writer = MaxMind::DB::Writer::Tree->new(
# ip_version => 6,
# record_size => 24,
# ip_version => 6,
# database_type => 'MaxMind DB No IPv4 Search Tree',
# languages => ['en'],
# description => {
# en => 'MaxMind DB No IPv4 Search Tree',
# },
# remove_reserved_networks => 0,
# root_data_type => 'utf8_string',
# map_key_type_callback => sub { {} },
# );
#
# my $subnet = Net::Works::Network->new_from_string( string => '::/64' );
# $writer->insert_network( $subnet, $subnet->as_string() );
#
# open my $output_fh, '>', "$Dir/MaxMind-DB-no-ipv4-search-tree.mmdb";
# $writer->write_tree($output_fh);
# close $output_fh;
#
# return;
# }
# The point of this database is to provide something where we can test looking
# up a single value. In other words, each IP address points to a non-compound
# value, a string rather than a map or array.
# sub write_no_map_db {
# my $subnets = shift;
#
# my $writer = MaxMind::DB::Writer::Tree->new(
# ip_version => 4,
# record_size => 24,
# database_type => 'MaxMind DB String Value Entries',
# languages => ['en'],
# description => {
# en =>
# 'MaxMind DB String Value Entries (no maps or arrays as values)',
# },
# root_data_type => 'utf8_string',
# map_key_type_callback => sub { {} },
# );
#
# for my $subnet ( @{$subnets} ) {
# $writer->insert_network( $subnet, $subnet->as_string() );
# }
#
# open my $output_fh, '>', "$Dir/MaxMind-DB-string-value-entries.mmdb";
# $writer->write_tree($output_fh);
# close $output_fh;
#
# return;
# }
# sub write_test_serialization_data {
# my $serializer = MaxMind::DB::Writer::Serializer->new(
# map_key_type_callback => sub { 'utf8_string' } );
#
# $serializer->store_data( map => { long_key => 'long_value1' } );
# $serializer->store_data( map => { long_key => 'long_value2' } );
# $serializer->store_data( map => { long_key2 => 'long_value1' } );
# $serializer->store_data( map => { long_key2 => 'long_value2' } );
# $serializer->store_data( map => { long_key => 'long_value1' } );
# $serializer->store_data( map => { long_key2 => 'long_value2' } );
#
# open my $fh, '>', 'maps-with-pointers.raw';
# print {$fh} ${ $serializer->buffer() }
# or die "Cannot write to maps-with-pointers.raw: $!";
# close $fh;
#
# return;
# }
# sub write_db_with_metadata_pointers {
# my $repeated_string = 'Lots of pointers in metadata';
# my $writer = MaxMind::DB::Writer::Tree->new(
# ip_version => 6,
# record_size => 24,
# map_key_type_callback => sub { 'utf8_string' },
# database_type => $repeated_string,
# languages => [ 'en', 'es', 'zh' ],
# description => {
# en => $repeated_string,
# es => $repeated_string,
# zh => $repeated_string,
# },
#
# );
#
# _populate_all_networks( $writer, {} );
#
# open my $fh, '>', 'MaxMind-DB-test-metadata-pointers.mmdb';
#
# $writer->write_tree($fh);
#
# close $fh;
# }
main();
================================================
FILE: LICENSE
================================================
Apache License
Version 2.0, January 2004
https://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README-Hive.md
================================================
Hive
====
The SerDe (it's really only a Deserializer) can be used present an Apache HTTPD logfile as a table in Hive.
This is an annotated example on how you could make the logfiles directly accessible through Hive.
First we must ensure that Hive has the right jar file available. This can be either using the ADD JAR option in the Hive Cli
or by installing it on the cluster.
ADD JAR target/httpdlog-serde-*-udf.jar;
We can now define an external table with column types are STRING, BIGINT and DOUBLE.
CREATE EXTERNAL TABLE nbasjes.clicks (
ip STRING
,timestamp BIGINT
,useragent STRING
,referrer STRING
,bui STRING
,screenHeight BIGINT
,screenWidth BIGINT
)
Of course we must specify the class name of the Deserializer that does the heavy lifting.
ROW FORMAT SERDE 'nl.basjes.parse.apachehttpdlog.ApacheHttpdlogDeserializer'
The big part of the config lies in the SERDEPROPERTIES.
There are currently 4 types of options you can/must put in there:
- "logformat" = "[Apache httpd logformat]"
- "field:[columnname]" = "[Field]"
- "map:[field]" = "[new type]"
- "load:[classname that implements Dissector]" = "[initialization string send to the initializeFromSettingsParameter method]"
Note that the order of various settings in the SERDEPROPERTIES is irrelevant.
WITH SERDEPROPERTIES (
**"logformat" = "[Apache httpd logformat]"**
This is the Logformat specification straight from the apache httpd config file.
"logformat" = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\" %T %V"
**"field:[columnname]" = "[Field]"**
For each column this type of property is needed for the system to know where to get the content from.
,"field:timestamp" = "TIME.EPOCH:request.receive.time.epoch"
,"field:ip" = "IP:connection.client.host"
,"field:useragent" = "HTTP.USERAGENT:request.user-agent"
**"map:[field]" = "[new type]"**
Only used when mapping a specific field to a different type.
,"map:request.firstline.uri.query.g"="HTTP.URI"
,"map:request.firstline.uri.query.r"="HTTP.URI"
,"field:referrer" = "STRING:request.firstline.uri.query.g.query.referrer"
,"field:bui" = "HTTP.COOKIE:request.cookies.bui"
**"load:[classname that implements Dissector]" = "[initialization string send to the initializeFromSettingsParameter method]"**
Only used when there is a custom Dissector implementation that needs to be loaded in addition to the regular Dissectors.
,"load:nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector" = "x"
,"map:request.firstline.uri.query.s" = "SCREENRESOLUTION"
,"field:screenHeight" = "SCREENHEIGHT:request.firstline.uri.query.s.height"
,"field:screenWidth" = "SCREENWIDTH:request.firstline.uri.query.s.width"
)
Finally we define that this is stored as a TEXTFILE and where the files are located.
STORED AS TEXTFILE
LOCATION "/user/nbasjes/clicks";
Complete example
====
ADD JAR target/httpdlog-serde-*-udf.jar;
CREATE EXTERNAL TABLE nbasjes.clicks (
ip STRING
,timestamp BIGINT
,useragent STRING
,referrer STRING
,bui STRING
,screenHeight BIGINT
,screenWidth BIGINT
)
ROW FORMAT SERDE 'nl.basjes.parse.apachehttpdlog.ApacheHttpdlogDeserializer'
WITH SERDEPROPERTIES (
"logformat" = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\" %T %V"
,"field:timestamp" = "TIME.EPOCH:request.receive.time.epoch"
,"field:ip" = "IP:connection.client.host"
,"field:useragent" = "HTTP.USERAGENT:request.user-agent"
,"map:request.firstline.uri.query.g"="HTTP.URI"
,"map:request.firstline.uri.query.r"="HTTP.URI"
,"field:referrer" = "STRING:request.firstline.uri.query.g.query.referrer"
,"field:bui" = "HTTP.COOKIE:request.cookies.bui"
,"load:nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector" = "x"
,"map:request.firstline.uri.query.s" = "SCREENRESOLUTION"
,"field:screenHeight" = "SCREENHEIGHT:request.firstline.uri.query.s.height"
,"field:screenWidth" = "SCREENWIDTH:request.firstline.uri.query.s.width"
)
STORED AS TEXTFILE
LOCATION "/user/nbasjes/clicks";
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README-Java.md
================================================
Apache HTTPD logparser
===
This is a Logparsing framework intended to make parsing Apache HTTPD logfiles much easier.
The basic idea is that you should be able to have a parser that you can construct by simply
telling it with what configuration options the line was written.
Usage (Java)
===
For the Java API there is an annotation based parser.
First you put something like this in your pom.xml file:
<dependency>
<groupId>nl.basjes.parse.httpdlog</groupId>
<artifactId>httpdlog-parser</artifactId>
<version>6.0.0</version>
</dependency>
I assume we have a logformat variable that looks something like this:
String logformat = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"";
**Step 1: What CAN we get from this line?**
To figure out what values we CAN get from this line we instantiate the parser with a dummy class
that does not have ANY @Field annotations. The "Object" class will do just fine for this purpose.
Parser<Object> dummyParser = new HttpdLoglineParser<Object>(Object.class, logformat);
List<String> possiblePaths = dummyParser.getPossiblePaths();
for (String path: possiblePaths) {
System.out.println(path);
}
You will get a list that looks something like this:
IP:connection.client.host
NUMBER:connection.client.logname
STRING:connection.client.user
TIME.STAMP:request.receive.time
TIME.DAY:request.receive.time.day
TIME.MONTHNAME:request.receive.time.monthname
TIME.MONTH:request.receive.time.month
TIME.YEAR:request.receive.time.year
TIME.HOUR:request.receive.time.hour
TIME.MINUTE:request.receive.time.minute
TIME.SECOND:request.receive.time.second
TIME.MILLISECOND:request.receive.time.millisecond
TIME.ZONE:request.receive.time.timezone
HTTP.FIRSTLINE:request.firstline
HTTP.METHOD:request.firstline.method
HTTP.URI:request.firstline.uri
HTTP.QUERYSTRING:request.firstline.uri.query
STRING:request.firstline.uri.query.*
HTTP.PROTOCOL:request.firstline.protocol
HTTP.PROTOCOL.VERSION:request.firstline.protocol.version
STRING:request.status.last
BYTESCLF:response.body.bytes
HTTP.URI:request.referer
HTTP.QUERYSTRING:request.referer.query
STRING:request.referer.query.*
HTTP.USERAGENT:request.user-agent
Now some of these lines contain a * .
This is a wildcard that can be replaced with any 'name' if you need a specific value.
You can also leave the '*' and get everything that is found in the actual log line.
**Step 2 Create the receiving POJO**
We need to create the receiving record class that is simply a POJO that does not need any interface or inheritance.
In this class we create setters that will be called when the specified field has been found in the line.
So we can now add to this class a setter that simply receives a single value:
@Field("IP:connection.client.host")
public void setIP(final String value) {
ip = value;
}
If we really want the name of the field we can also do this
@Field("STRING:request.firstline.uri.query.img")
public void setQueryImg(final String name, final String value) {
results.put(name, value);
}
This latter form is very handy because this way we can obtain all values for a wildcard field
@Field("STRING:request.firstline.uri.query.*")
public void setQueryStringValues(final String name, final String value) {
results.put(name, value);
}
Or a combination of the above examples where you specify multiple field patterns
@Field({"IP:connection.client.host",
"STRING:request.firstline.uri.query.*"})
public void setValue(final String name, final String value) {
results.put(name, value);
}
In some cases you may not want to have empty/null values so starting with version 5.0 you can specify a setterPolicy:
@Field(value = "STRING:request.firstline.uri.query.*", setterPolicy = NOT_NULL)
The 3 possible values for the setterPolicy flag are:
ALWAYS : Call the setter for all values: Normal, Empty and NULL.
NOT_NULL : Call the setter for values: Normal and Empty, but not for NULL values.
NOT_EMPTY : Call the setter for values: Normal, but not for Empty and NULL values.
*Notes about the setters*
- Only if a value exists in the actual logline the setter will be called (mainly relevant if you want to get a specific query param or cookie).
- If you specifiy the same field on several setters then each of these setters will be called.
- There is NO guarantee about the order the setters will be called.
Have a look at the 'examples/pojo' directory for a working example.
**Step 3 Use the parser in your application.**
You create an instance of the parser
Parser<MyRecord> parser = new HttpdLoglineParser<MyRecord>(MyRecord.class, logformat);
And then call the parse method repeatedly for each line.
There are two ways to do this:
1) Let the parser create and a new instance of "MyRecord" for each parsed line (think about the GC consequences!!):
MyRecord record = parser.parse(logline);
2) Reuse the same instance.
So you do this only once:
MyRecord record = new MyRecord();
And then for each logline:
record.clear(); // Which is up to you to implement to 'reset' the record instance to it's initial/empty state.
parser.parse(record, logline);
Project Lombok
===
In case you like to use project Lombok to generate your getters and setters then using the annotations looks something like this:
@Getter @Setter(onMethod=@__(@Field("HTTP.COOKIE:request.cookies.foo"))) private String foo = null;
To avoid weird effects please install the "Lombok Plugin" in IntelliJ IDEA to use this.
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README-Pig.md
================================================
Abandoned!
===
Version 5.8 is the last version to support Apache Pig.
The last release of Apache Pig was in 2017 and right now (2023) the tool is effectively no longer used by anyone.
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README-geoip.md
================================================
Dissect IP using GeoIP2 information
===
This project also contains a dissector that uses the [MaxMind](https://www.maxmind.com) GeoIP2 data to
dissect IP addresses into things like Country, City, ASN, etc.
Where are the datafiles?
---
Simple: I didn't include them.
The data is owned by MaxMind and in order to use it you must either purchase a license for 'accurate' GeoIP2
data or download a 'slightly less accurate' free GeoLite2 version.
Also adding these files would make the repo very big.
See https://dev.maxmind.com/ for the both the paid GeoIP2 and the free GeoLite2 downloadable databases.
I personally install and run the geoipupdate tool.
https://dev.maxmind.com/geoip/geoipupdate/
The datafiles I usually work with:
/var/lib/GeoIP/GeoLite2-City.mmdb
/var/lib/GeoIP/GeoLite2-Country.mmdb
/var/lib/GeoIP/GeoIP2-ISP.mmdb
/var/lib/GeoIP/GeoLite2-ASN.mmdb
You can get some of those by installing geoipupdate tool with the config file /etc/GeoIP.conf
# The following UserId and LicenseKey are required placeholders:
UserId 999999
LicenseKey 000000000000
ProductIds GeoLite2-City GeoLite2-Country GeoLite2-ASN
How do I use it?
===
Currently there are 4 dissectors available
ASN
---
* Class: nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPASNDissector
* Input: Needs the path to the GeoLite2-ASN.mmdb to function.
* Output: ASN number and organization.
ISP
---
* Class: nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPISPDissector
* Input: Needs the path to the GeoIP2-ISP.mmdb or GeoLite2-ISP.mmdb to function.
* Output: ASN number and organization, ISP name and organization.
Country
---
* Class: nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPCountryDissector
* Input: Needs the path to the GeoIP2-Country.mmdb or GeoLite2-Country.mmdb to function.
* Output: Information about continent and country.
City
---
* Class: nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPCityDissector
* Input: Needs the path to the GeoIP2-City.mmdb or GeoLite2-City.mmdb to function.
* Output: Information about continent, country, subdivision, city, postalcode and latitude/longitude.
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: README.md
================================================
Apache HTTPD & NGINX access log parser
======================================
[](https://github.com/nielsbasjes/logparser/actions)
[](https://app.codecov.io/gh/nielsbasjes/logparser)
[](https://www.apache.org/licenses/LICENSE-2.0.html)
[](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22nl.basjes.parse.httpdlog%22)
[](https://github.com/sponsors/nielsbasjes)
[](https://www.paypal.me/nielsbasjes)
This is a Logparsing framework intended to make parsing [Apache HTTPD](https://httpd.apache.org/) and [NGINX](https://nginx.org/) access log files much easier.
The basic idea is that you should be able to have a parser that you can construct by simply
telling it with what configuration options the line was written.
These configuration options are the schema of the access loglines.
So we are using the LogFormat that wrote the file as the input parameter for the parser that reads the same file.
In addition to the config options specified in the Apache HTTPD manual under
[Custom Log Formats](https://httpd.apache.org/docs/current/mod/mod_log_config.html) the following are also recognized:
* common
* combined
* combinedio
* referer
* agent
For Nginx the log_format tokens are specified [here](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) and [here](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables).
**Special notes about the Apache HTTPD token %{format}t**
===
Quote from [Apache HTTPD manual](https://httpd.apache.org/docs/current/mod/mod_log_config.html#formats)
%{format}t: The time, in the form given by format, which should be in strftime(3) format. (potentially localized)
* **Version 2.5 and before**:
It cannot be extracted. A simple workaround for this limitation: replace the **%{...}t** with **%{timestamp}i** .
You will then get this timestamp field as if it was a request header: HTTP.HEADER:request.header.timestamp
* **Version 2.6 and newer**: You will receive it as a textual *TIME.LOCALIZEDSTRING:request.header.time* which cannot be extracted any further.
* **Version 3.0 and newer**: Support for parsing the customized time as long as all elements can be mapped to fields supported by joda-time.
This means that many fields are supported, but not all. Check the implementation in the [StrfTimeStampDissector](../v3.1/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeStampDissector.java#L140) class to see which are and are not supported.
* **Version 4.0 and newer**: Switched to parsing using native java 8 time library supports a few fields differently. See [StrfTimeToDateTimeFormatter](/httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeToDateTimeFormatter.java#L119).
**Limitation**: Only a single %{format}t entry is supported per line.
Examples as described in the LogFormat [examples section](https://httpd.apache.org/docs/current/mod/mod_log_config.html#examples)
of the Apache HTTPD manual cannot be parsed.
You can use the %{format}t directive multiple times to build up a time format using the extended format tokens like msec_frac:
Timestamp including milliseconds
"%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t"
In this case where all %{format}t fields are only separated by fixed text you can rewrite this example like this
"%{%d/%b/%Y %T}t.%{msec_frac}t %{%z}t"
"%{%d/%b/%Y %T.msec_frac %z}t"
Although the latter is NOT supported by Apache HTTPD this IS supported by this logparser so the above works as expected.
Analyze almost anything
===
I wrote this parser for practical reallife situations. In reality a lot happens that is not allowed when looking at the
official specifications, yet in production they do happen.
So several of the key parts in this parser try to recover from bad data where possible and thus allow to extract as
much useful information as possible even if the data is not valid.
Important examples of this are invalid encoding characters and chopped multibyte encoded characters that are both
extracted as best as possible.
If you have a real logline that causes a parse error then I kindly request you to submit this line, the logformat and
the field that triggered the error as a bug report.
Pre built versions
===
Prebuilt versions have been deployed to maven central so using it in a project is as simple as adding a dependency.
So using it in a Java based project is as simple as adding this to your dependencies
<dependency>
<groupId>nl.basjes.parse.httpdlog</groupId>
<artifactId>httpdlog-parser</artifactId>
<version>6.0.0</version>
</dependency>
Note that starting with version 6.0.0 Java 21 is needed as a runtime because of several dependencies that have updated to that.
Building
===
Simply type : mvn package
and the whole thing should build.
Java, Apache {Hadoop, Hive, Drill, Flink, Beam}
===
I'm a big user of bigdata tools like Apache Hadoop, Hive, etc. .
So in here are also a Hadoop inputformat and a Hive/HCatalog Serde that are wrappers around this library.
Usage (Overview)
===
The framework needs two things:
* The format specification in which the logfile was written (straight from the original apache httpd config file).
* The identifiers for the fields that you want.
To obtain all the identifiers the system CAN extract from the specified logformat a separate
developer call exists in various languages that allows you to get the list of all possible values.
Languages and Tools
===
The languages that are supported in this version:
* [Java](README-Java.md)
Prebuilt plugins for these are provided in the distribution:
* [Apache Hive](README-Hive.md)
For tools like Apache Flink and Beam there is only example code that is also used to verify that the build
still works on those systems.
* [Apache Flink](examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink)
* [Apache Beam](examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam)
Tools that ship a version of this parser in their distribution
* ~~[Apache Pig](https://github.com/apache/pig/blob/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/apachelog/LogFormatLoader.java)~~
* [Apache Drill](https://drill.apache.org/docs/httpd-format-plugin/)
Internal structure and type remapping
===
The basic model of this system is a tree.
Each node in the tree has a 'type' and a 'name'.
The 'type' is really a 'what is the format of this string' indicator. Because there are many more of those kinds of types than your average String or Long you will see a lot of different names.
The 'name' is the "breadbrumb" towards the point in the tree where this is located.
A 'Dissector' is a class that can cut a specific type (format) into a bunch of new parts that each extend the base name and have their own type.
Because an internal parser is constructed at the start of running a parser this tree has some dynamic properties.
To start only the tree is constructed for the elements actually requested. This is done to avoid 'dissecting' something that is not wanted.
So the parser will have a different structure depending on the requested output.
These dynamic properties also allow 'mapping a field to a different type'. Lets illustrate what this is by looking at the most common usecase.
Assume you are trying to parse the logline for a pixel that was written by a webanalytics product. In that scenario it is common that the URL is that of a pixel and one of the query string parameters contains the actual URL. Now by default a querystring parameter gets the type STRING (which really means that is is arbitrary and cannot be dissected any further). Using this remapping (see API details per language) we can now say that a specific query string parameter really has the type HTTP.URL. As a consequence the system can now continue dissecting this specific query string parameter into things like the host, port and query string parameters.
All that is needed to map the 'g' and 'r' parameters so they are dissected further is this:
Java: Call these against the parser instance right after construction
parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI", Casts.STRING_ONLY);
parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI", Casts.STRING_ONLY);
Hive: Add these to the SERDEPROPERTIES
"map:request.firstline.uri.query.g"="HTTP.URI",
"map:request.firstline.uri.query.r"="HTTP.URI",
Special Dissectors
===
**mod_unique_id**
If you have a log field / request header that gets filled using mod_unique_id you can now peek inside
the values that were used to construct this.
**NOTE: https://httpd.apache.org/docs/current/mod/mod_unique_id.html clearly states**
it should be emphasized that applications should not dissect the encoding.
Applications should treat the entire encoded UNIQUE_ID as an opaque token,
which can be compared against other UNIQUE_IDs for equality only.
When you choose to ignore the clear 'should not' statement then simply add
a type remapping to map the field to the type *MOD_UNIQUE_ID*
**GeoIP parsing**
Head for the separate [README](README-geoip.md) file for information about this dissector.
Parsing problems with Jetty generated logfiles
==============================================
In Jetty there is the option to create a logfile in what they call the NCSARequestLog format.
It was found that (historically) this had two formatting problems which cause parse errors:
1. If the useragent is missing the empty value is logged with an extra ' ' after it.
The fix for this in Jetty was committed on 2016-07-27 in the Jetty 9.3.x and 9.4.x branches
2. Before jetty-9.2.4.v20141103 if there is no user available the %u field is logged as " - "
(i.e. with two extra spaces around the '-').
To workaround these problems you can easily start the parser with a two line logformat:
ENABLE JETTY FIX
%h %l %u %t \"%r\" %>s %b "%{Referer}i" "%{User-Agent}i" %D
This *ENABLE JETTY FIX* is a 'magic' value that causes the underlying parser to enable the workaround for both of these problems.
In order for this to work correctly the useragent field must look exactly like this: *"%{User-Agent}i"*
Donations
===
If this project has business value for you then don't hesitate to support me with a small donation.
[](https://github.com/sponsors/nielsbasjes)
[](https://www.paypal.me/nielsbasjes)
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: _config.yml
================================================
theme: jekyll-theme-cayman
================================================
FILE: devtools/docker/Dockerfile
================================================
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
FROM ubuntu:26.04
WORKDIR /root
ENV INSIDE_DOCKER Yes
ARG DEBIAN_FRONTEND=noninteractive
WORKDIR /root
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
#####
# Disable suggests/recommends
#####
RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
###
# Update and install common packages
###
RUN apt -q update \
&& apt install -y software-properties-common apt-utils apt-transport-https ca-certificates \
&& add-apt-repository -y ppa:deadsnakes/ppa
RUN apt-get -q install -y --no-install-recommends \
bash-completion \
build-essential \
bzip2 \
wget \
curl \
docker.io \
git \
gnupg-agent \
rsync \
sudo \
vim \
locales \
wget \
time \
ruby \
openjdk-8-jdk \
openjdk-11-jdk \
openjdk-17-jdk
###
# Set the locale ( see https://stackoverflow.com/a/28406007/114196 )
###
RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
locale-gen
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
# --------------------------------
# Install Maven
ENV MAVEN_VERSION=3.8.3
RUN mkdir -p /usr/local/apache-maven
RUN cd /usr/local/ && wget "https://www.apache.org/dyn/closer.lua?action=download&filename=/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz" -O "apache-maven-${MAVEN_VERSION}-bin.tar.gz"
RUN cd /usr/local/ && tar xzf apache-maven-${MAVEN_VERSION}-bin.tar.gz --strip-components 1 -C /usr/local/apache-maven
ENV M2_HOME /usr/local/apache-maven
ENV PATH ${M2_HOME}/bin:${PATH}
# Avoid out of memory errors in builds
ENV MAVEN_OPTS -Xms256m -Xmx512m
# Install command line completion for maven
RUN wget https://raw.githubusercontent.com/juven/maven-bash-completion/master/bash_completion.bash -O /etc/bash_completion.d/maven
# --------------------------------
# Install shellcheck
RUN cd /usr/local/bin && \
wget https://github.com/koalaman/shellcheck/releases/download/stable/shellcheck-stable.linux.x86_64.tar.xz && \
tar xJf shellcheck-stable.linux.x86_64.tar.xz && \
mv shellcheck-stable/shellcheck . && \
rm -rf shellcheck-stable*
# --------------------------------
# Install Hugo
ENV HUGO_VERSION=0.89.0
RUN cd /usr/local/bin && \
wget https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_${HUGO_VERSION}_Linux-64bit.tar.gz && \
tar xzf hugo_*.tar.gz
# --------------------------------
# Add a welcome message and environment checks.
RUN mkdir /scripts
ADD *.sh /scripts/
RUN chmod 755 /scripts/*.sh
# --------------------------------
# For serving the documentation site
EXPOSE 1313
================================================
FILE: devtools/docker/bashcolors.sh
================================================
#!/bin/bash
#
# Yet Another UserAgent Analyzer
# Copyright (C) 2013-2022 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#https://wiki.archlinux.org/index.php/Color_Bash_Prompt
# Reset
export Color_Off='\e[0m' # Text Reset
# Regular Colors
export Black='\e[0;30m' # Black
export Red='\e[0;31m' # Red
export Green='\e[0;32m' # Green
export Yellow='\e[0;33m' # Yellow
export Blue='\e[0;34m' # Blue
export Purple='\e[0;35m' # Purple
export Cyan='\e[0;36m' # Cyan
export White='\e[0;37m' # White
# Bold
export BBlack='\e[1;30m' # Black
export BRed='\e[1;31m' # Red
export BGreen='\e[1;32m' # Green
export BYellow='\e[1;33m' # Yellow
export BBlue='\e[1;34m' # Blue
export BPurple='\e[1;35m' # Purple
export BCyan='\e[1;36m' # Cyan
export BWhite='\e[1;37m' # White
# Underline
export UBlack='\e[4;30m' # Black
export URed='\e[4;31m' # Red
export UGreen='\e[4;32m' # Green
export UYellow='\e[4;33m' # Yellow
export UBlue='\e[4;34m' # Blue
export UPurple='\e[4;35m' # Purple
export UCyan='\e[4;36m' # Cyan
export UWhite='\e[4;37m' # White
# Background
export On_Black='\e[40m' # Black
export On_Red='\e[41m' # Red
export On_Green='\e[42m' # Green
export On_Yellow='\e[43m' # Yellow
export On_Blue='\e[44m' # Blue
export On_Purple='\e[45m' # Purple
export On_Cyan='\e[46m' # Cyan
export On_White='\e[47m' # White
# High Intensity
export IBlack='\e[0;90m' # Black
export IRed='\e[0;91m' # Red
export IGreen='\e[0;92m' # Green
export IYellow='\e[0;93m' # Yellow
export IBlue='\e[0;94m' # Blue
export IPurple='\e[0;95m' # Purple
export ICyan='\e[0;96m' # Cyan
export IWhite='\e[0;97m' # White
# Bold High Intensity
export BIBlack='\e[1;90m' # Black
export BIRed='\e[1;91m' # Red
export BIGreen='\e[1;92m' # Green
export BIYellow='\e[1;93m' # Yellow
export BIBlue='\e[1;94m' # Blue
export BIPurple='\e[1;95m' # Purple
export BICyan='\e[1;96m' # Cyan
export BIWhite='\e[1;97m' # White
# High Intensity backgrounds
export On_IBlack='\e[0;100m' # Black
export On_IRed='\e[0;101m' # Red
export On_IGreen='\e[0;102m' # Green
export On_IYellow='\e[0;103m' # Yellow
export On_IBlue='\e[0;104m' # Blue
export On_IPurple='\e[0;105m' # Purple
export On_ICyan='\e[0;106m' # Cyan
export On_IWhite='\e[0;107m' # White
================================================
FILE: devtools/docker/build_env_checks.sh
================================================
#!/bin/bash
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# -------------------------------------------------------
function showWelcome {
# http://patorjk.com/software/taag/#p=display&f=Slant&t=LogParser%20Builder
cat << "Welcome-message"
__ ____ ____ _ __ __
/ / ____ ____ _/ __ \____ ______________ _____ / __ )__ __(_) /___/ /__ _____
/ / / __ \/ __ `/ /_/ / __ `/ ___/ ___/ _ \/ ___/ / __ / / / / / / __ / _ \/ ___/
/ /___/ /_/ / /_/ / ____/ /_/ / / (__ ) __/ / / /_/ / /_/ / / / /_/ / __/ /
/_____/\____/\__, /_/ \__,_/_/ /____/\___/_/ /_____/\__,_/_/_/\__,_/\___/_/
/____/
This is the standard LogParser build environment.
In here all tools needed to run a build are present.
Welcome-message
}
# -------------------------------------------------------
function showAbort {
# http://patorjk.com/software/taag/#p=display&f=Doom&t=Aborting...
cat << "Abort-message"
___ _ _ _
/ _ \| | | | (_)
/ /_\ \ |__ ___ _ __| |_ _ _ __ __ _
| _ | '_ \ / _ \| '__| __| | '_ \ / _` |
| | | | |_) | (_) | | | |_| | | | | (_| |_ _ _
\_| |_/_.__/ \___/|_| \__|_|_| |_|\__, (_|_|_)
__/ |
|___/
Abort-message
}
# -------------------------------------------------------
function failIfUserIsRoot {
if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
then
cat <<End-of-message
Apparently you are inside this docker container as the user root.
Putting it simply:
This should not occur.
Known possible causes of this are:
1) Running this script as the root user ( Just don't )
2) Running an old docker version ( upgrade to 1.4.1 or higher )
End-of-message
showAbort
logout
fi
}
# -------------------------------------------------------
# Configurable low water mark in GiB
MINIMAL_MEMORY_GiB=2
function warnIfLowMemory {
MINIMAL_MEMORY=$((MINIMAL_MEMORY_GiB*1024*1024)) # Convert to KiB
INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
if [ $((INSTALLED_MEMORY)) -le $((MINIMAL_MEMORY)) ];
then
cat << "End-of-message"
_ ___ ___
| | | \/ |
| | _____ __ | . . | ___ _ __ ___ ___ _ __ _ _
| | / _ \ \ /\ / / | |\/| |/ _ \ '_ ` _ \ / _ \| '__| | | |
| |___| (_) \ V V / | | | | __/ | | | | | (_) | | | |_| |
\_____/\___/ \_/\_/ \_| |_/\___|_| |_| |_|\___/|_| \__, |
__/ |
|___/
End-of-message
cat << "End-of-message"
Your system is running on very little memory.
This means it may work but it wil most likely be slower than needed.
If you are running this via boot2docker you can simply increase
the available memory to atleast ${MINIMAL_MEMORY_GiB} GiB (you have $((INSTALLED_MEMORY/(1024*1024))) GiB )
End-of-message
fi
}
# -------------------------------------------------------
showWelcome
warnIfLowMemory
failIfUserIsRoot
# -------------------------------------------------------
================================================
FILE: devtools/docker/configure-for-user.sh
================================================
#!/bin/bash
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Native Linux (direct or via sudo)
USER_NAME=$1
USER_ID=$2
GROUP_ID=$3
groupadd --non-unique -g "${GROUP_ID}" "${USER_NAME}"
useradd -g "${GROUP_ID}" -u "${USER_ID}" -k /root -m "${USER_NAME}"
chown "${USER_NAME}" "/home/${USER_NAME}"
{
echo "export HOME=/home/${USER_NAME}"
echo "export USER=${USER_NAME}"
echo '. /scripts/env.sh'
} >> "/home/${USER_NAME}/.bashrc"
VBOXSF_GROUP_LINE=$4
if [[ -n "${VBOXSF_GROUP_LINE}" ]];
then
echo "${VBOXSF_GROUP_LINE}" >> /etc/group
usermod -aG vboxsf "${USER_NAME}"
fi
echo "${USER_NAME} ALL=(ALL) NOPASSWD: ALL" >> "/etc/sudoers.d/${USER_NAME}"
================================================
FILE: devtools/docker/env.sh
================================================
#!/bin/bash
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
. "/scripts/build_env_checks.sh"
. "/scripts/bashcolors.sh"
export JDK_VERSION="???"
function __INTERNAL__SwitchJDK {
JDK=$1
echo -e "${IRed}${On_Black}Setting JDK to version ${JDK}${Color_Off}"
sudo update-java-alternatives --set $(update-java-alternatives -l | fgrep "${JDK}" | sed 's@ \+@ @g' | cut -d' ' -f1);
export JAVA_HOME=$(update-java-alternatives -l | fgrep "${JDK}" | sed 's@ \+@ @g' | cut -d' ' -f3);
# export JDK_VERSION="JDK ${JDK}"
}
echo "Use switch-jdk8, switch-jdk11 or switch-jdk17 to select the desired JDK to build with."
alias switch-jdk8="__INTERNAL__SwitchJDK 1.8.0; export JDK_VERSION=JDK-8"
alias switch-jdk11="__INTERNAL__SwitchJDK 1.11.0; export JDK_VERSION=JDK-11"
alias switch-jdk17="__INTERNAL__SwitchJDK 1.17.0 ; export JDK_VERSION=JDK-17"
#alias switch-jdk13="__INTERNAL__SwitchJDK latest; export JDK_VERSION=JDK-13"
switch-jdk11
. "/usr/lib/git-core/git-sh-prompt"
export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][Yauaa Builder \['${BWhite}${On_Blue}'\]<'\${JDK_VERSION}'>\['${IWhite}${On_Red}'\]]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ '
alias documentation-serve="cd ~/yauaa/documentation && hugo server -b http://localhost --bind=0.0.0.0"
================================================
FILE: devtools/docker/prompt.sh
================================================
#!/bin/bash
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#https://wiki.archlinux.org/index.php/Color_Bash_Prompt
# Reset
Color_Off='\e[0m' # Text Reset
# Regular Colors
#Black='\e[0;30m' # Black
#Red='\e[0;31m' # Red
#Green='\e[0;32m' # Green
#Yellow='\e[0;33m' # Yellow
#Blue='\e[0;34m' # Blue
#Purple='\e[0;35m' # Purple
Cyan='\e[0;36m' # Cyan
#White='\e[0;37m' # White
# Bold
#BBlack='\e[1;30m' # Black
#BRed='\e[1;31m' # Red
#BGreen='\e[1;32m' # Green
#BYellow='\e[1;33m' # Yellow
#BBlue='\e[1;34m' # Blue
#BPurple='\e[1;35m' # Purple
#BCyan='\e[1;36m' # Cyan
#BWhite='\e[1;37m' # White
# Underline
#UBlack='\e[4;30m' # Black
#URed='\e[4;31m' # Red
#UGreen='\e[4;32m' # Green
#UYellow='\e[4;33m' # Yellow
#UBlue='\e[4;34m' # Blue
#UPurple='\e[4;35m' # Purple
#UCyan='\e[4;36m' # Cyan
#UWhite='\e[4;37m' # White
# Background
On_Black='\e[40m' # Black
On_Red='\e[41m' # Red
#On_Green='\e[42m' # Green
#On_Yellow='\e[43m' # Yellow
#On_Blue='\e[44m' # Blue
#On_Purple='\e[45m' # Purple
#On_Cyan='\e[46m' # Cyan
#On_White='\e[47m' # White
# High Intensity
#IBlack='\e[0;90m' # Black
#IRed='\e[0;91m' # Red
#IGreen='\e[0;92m' # Green
#IYellow='\e[0;93m' # Yellow
IBlue='\e[0;94m' # Blue
#IPurple='\e[0;95m' # Purple
#ICyan='\e[0;96m' # Cyan
IWhite='\e[0;97m' # White
# Bold High Intensity
#BIBlack='\e[1;90m' # Black
#BIRed='\e[1;91m' # Red
BIGreen='\e[1;92m' # Green
#BIYellow='\e[1;93m' # Yellow
BIBlue='\e[1;94m' # Blue
BIPurple='\e[1;95m' # Purple
#BICyan='\e[1;96m' # Cyan
#BIWhite='\e[1;97m' # White
# High Intensity backgrounds
#On_IBlack='\e[0;100m' # Black
#On_IRed='\e[0;101m' # Red
#On_IGreen='\e[0;102m' # Green
#On_IYellow='\e[0;103m' # Yellow
#On_IBlue='\e[0;104m' # Blue
#On_IPurple='\e[0;105m' # Purple
#On_ICyan='\e[0;106m' # Cyan
#On_IWhite='\e[0;107m' # White
export PS1='\['${IBlue}${On_Black}'\] \u@\['${IWhite}${On_Red}'\][LogParser Builder]\['${IBlue}${On_Black}'\]:\['${Cyan}${On_Black}'\]\w$(declare -F __git_ps1 &>/dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ '
================================================
FILE: devtools/logformat.conf
================================================
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2023 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# For testing (almost) all log format specifiers
LogFormat "\"%%\" \"%a\" \"%{c}a\" \"%A\" \"%B\" \"%b\" \"%D\" \"%f\" \"%h\" \"%H\" \"%k\" \"%l\" \"%L\" \"%m\" \"%p\" \"%{canonical}p\" \"%{local}p\" \"%{remote}p\" \"%P\" \"%{pid}P\" \"%{tid}P\" \"%{hextid}P\" \"%q\" \"%r\" \"%R\" \"%s\" \"%>s\" \"%t\" \"%{msec}t\" \"%{begin:msec}t\" \"%{end:msec}t\" \"%{usec}t\" \"%{begin:usec}t\" \"%{end:usec}t\" \"%{msec_frac}t\" \"%{begin:msec_frac}t\" \"%{end:msec_frac}t\" \"%{usec_frac}t\" \"%{begin:usec_frac}t\" \"%{end:usec_frac}t\" \"%T\" \"%u\" \"%U\" \"%v\" \"%V\" \"%X\" \"%I\" \"%O\" \"%{cookie}i\" \"%{set-cookie}o\" \"%{user-agent}i\" \"%{referer}i\"" logEverything
CustomLog "logs/fullaccess_log" logEverything
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "4880" "4880" "652" "/usr/share/httpd/noindex/index.html" "172.17.42.1" "HTTP/1.1" "0" "-" "VG9exZ0MX@uqta4OldejvQAAAAA" "GET" "80" "80" "80" "43417" "126" "126" "140597540726848" "140597540726848" "" "GET / HTTP/1.1" "httpd/unix-directory" "403" "403" "[21/Nov/2014:15:48:21 +0000]" "1416584901018" "1416584901018" "1416584901018" "1416584901018010" "1416584901018010" "1416584901018670" "018" "018" "018" "018010" "018010" "018670" "0" "-" "/" "172.17.0.2" "172.17.0.2" "+" "367" "5188" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "-"
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "0" "-" "302" "/usr/share/httpd/noindex/css/bootstrap.min.css" "172.17.42.1" "HTTP/1.1" "1" "-" "-" "GET" "80" "80" "80" "43417" "126" "126" "140597540726848" "140597540726848" "" "GET /css/bootstrap.min.css HTTP/1.1" "-" "304" "304" "[21/Nov/2014:15:48:21 +0000]" "1416584901087" "1416584901087" "1416584901087" "1416584901087115" "1416584901087115" "1416584901087417" "087" "087" "087" "087115" "087115" "087417" "0" "-" "/css/bootstrap.min.css" "172.17.0.2" "172.17.0.2" "+" "448" "180" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "http://172.17.0.2/"
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "0" "-" "373" "/usr/share/httpd/noindex/css/open-sans.css" "172.17.42.1" "HTTP/1.1" "0" "-" "-" "GET" "80" "80" "80" "43418" "127" "127" "140597540726848" "140597540726848" "" "GET /css/open-sans.css HTTP/1.1" "-" "304" "304" "[21/Nov/2014:15:48:21 +0000]" "1416584901087" "1416584901087" "1416584901087" "1416584901087430" "1416584901087430" "1416584901087803" "087" "087" "087" "087430" "087430" "087803" "0" "-" "/css/open-sans.css" "172.17.0.2" "172.17.0.2" "+" "444" "181" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "http://172.17.0.2/"
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "0" "-" "381" "/usr/share/httpd/noindex/images/apache_pb.gif" "172.17.42.1" "HTTP/1.1" "0" "-" "-" "GET" "80" "80" "80" "43419" "128" "128" "140597540726848" "140597540726848" "" "GET /images/apache_pb.gif HTTP/1.1" "-" "304" "304" "[21/Nov/2014:15:48:21 +0000]" "1416584901087" "1416584901087" "1416584901087" "1416584901087445" "1416584901087445" "1416584901087826" "087" "087" "087" "087445" "087445" "087826" "0" "-" "/images/apache_pb.gif" "172.17.0.2" "172.17.0.2" "+" "448" "180" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "http://172.17.0.2/"
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "0" "-" "269" "/usr/share/httpd/noindex/images/poweredby.png" "172.17.42.1" "HTTP/1.1" "1" "-" "-" "GET" "80" "80" "80" "43419" "128" "128" "140597540726848" "140597540726848" "" "GET /images/poweredby.png HTTP/1.1" "-" "304" "304" "[21/Nov/2014:15:48:21 +0000]" "1416584901091" "1416584901091" "1416584901091" "1416584901091601" "1416584901091601" "1416584901091870" "091" "091" "091" "091601" "091601" "091870" "0" "-" "/images/poweredby.png" "172.17.0.2" "172.17.0.2" "+" "448" "179" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "http://172.17.0.2/"
"%" "172.17.42.1" "172.17.42.1" "172.17.0.2" "213" "213" "448" "/var/www/html/ladkshjfkjasdhf" "172.17.42.1" "HTTP/1.1" "0" "-" "-" "GET" "80" "80" "80" "43482" "136" "136" "140597540726848" "140597540726848" "" "GET /ladkshjfkjasdhf HTTP/1.1" "-" "404" "404" "[21/Nov/2014:15:50:45 +0000]" "1416585045231" "1416585045231" "1416585045231" "1416585045231085" "1416585045231085" "1416585045231533" "231" "231" "231" "231085" "231085" "231533" "0" "-" "/ladkshjfkjasdhf" "172.17.0.2" "172.17.0.2" "+" "356" "429" "-" "-" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36" "-"
================================================
FILE: devtools/pom.xml
================================================
<?xml version="1.0" encoding="UTF-8"?>
<!--
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>parser-parent</artifactId>
<groupId>nl.basjes.parse</groupId>
<version>6.0.1-SNAPSHOT</version>
</parent>
<groupId>nl.basjes.parse.devtools</groupId>
<artifactId>devtools</artifactId>
<packaging>jar</packaging>
<name>Parser - Developer Tools</name>
<url>https://github.com/nielsbasjes/logparser</url>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<!-- Disable coverage analysis for this module -->
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: devtools/release.sh
================================================
#!/bin/bash
#
# Apache HTTPD & NGINX Access log parsing made easy
# Copyright (C) 2011-2025 Niels Basjes
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Release procedure.
# This uses the maven-release-plugin which has been configured to ONLY modify the local git repo.
# ----------------------------------------------------------------------------------------------------
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
echo "PWD: ${SCRIPTDIR}"
cd "${SCRIPTDIR}/.." || ( echo "This should not be possible" ; exit 1 )
# Working directory is now the root of the project
# ----------------------------------------------------------------------------------------------------
#https://wiki.archlinux.org/index.php/Color_Bash_Prompt
# Reset
export Color_Off='\e[0m' # Text Reset
# High Intensity
export IRed='\e[0;91m' # Red
export IYellow='\e[0;93m' # Yellow
export IBlue='\e[0;94m' # Blue
export IWhite='\e[0;97m' # White
# Bold High Intensity
export BIRed='\e[1;91m' # Red
export BIYellow='\e[1;93m' # Yellow
export BIGreen='\e[1;92m' # Green
export BIBlue='\e[1;94m' # Blue
function info() {
echo -e "${Color_Off}${IWhite}[${BIBlue}INFO${IWhite}] ${Color_Off}${1}"
}
function pass() {
echo -e "${Color_Off}${IWhite}[${BIGreen}PASS${IWhite}] ${Color_Off}${1}"
}
function warn() {
echo -e "${Color_Off}${IWhite}[${BIYellow}WARN${IWhite}] ${IYellow}${1}${Color_Off}"
}
function fail() {
echo -e "${Color_Off}${IWhite}[${BIRed}FAIL${IWhite}] ${IRed}${1}${Color_Off}"
}
function die() {
echo -e "${Color_Off}"
echo -e "${IWhite}[${BIRed}FAIL${IWhite}] ${IYellow}/========================================================================"
echo -e "${IWhite}[${BIRed}FAIL${IWhite}] ${IYellow}| ${BIRed} ---------->>> PROCESS WAS ABORTED <<<---------- ${IYellow}"
echo -e "${IWhite}[${BIRed}FAIL${IWhite}] ${IYellow}| ${BIRed} $* ${IYellow}"
echo -e "${IWhite}[${BIRed}FAIL${IWhite}] ${IYellow}\\========================================================================"
echo -e "${Color_Off}"
exit 1
}
# ----------------------------------------------------------------------------------------------------
# Pre flight checks
## Ensure all has been committed
info "Checking tree status"
if [[ -z $(git status -s) ]]
then
pass "Tree is clean"
else
git status
die "Tree is dirty, must commit everything"
fi
## Ensure we have all upstream updates (like patches from Renovate)
info "Checking up to date status"
git pull
gitPullStatus=$?
if [ ${gitPullStatus} -ne 0 ];
then
fail "We just received changes."
exit ${gitPullStatus}
else
pass "Everything is up to date."
fi
# ----------------------------------------------------------------------------------------------------
# Forcing a manual gpg signing action to ensure the password is known
(
cd /tmp || die "Unable to enter /tmp"
echo x > ReleaseProcess-$$.txt
gpg --clearsign ReleaseProcess-$$.txt
rm ReleaseProcess-$$.txt ReleaseProcess-$$.txt.asc
)
info "GPG workaround: Starting"
runGpgSignerInBackGround(){
while : ; do date ; echo "test" | gpg --clearsign ; sleep 10s ; done
}
runGpgSignerInBackGround > /dev/null 2>&1 &
GpgSignerPID=$!
info "GPG workaround: Running (PID=${GpgSignerPID})"
killSigner() {
info "GPG workaround: Killing (PID=${GpgSignerPID})"
kill ${GpgSignerPID}
info "GPG workaround: Killed"
}
trap killSigner EXIT
trap killSigner SIGINT
# ----------------------------------------------------------------------------------------------------
## Prepare the release: Make releasable version and make tag.
info "Doing release:prepare"
mvn release:prepare -B
prepareStatus=$?
if [ ${prepareStatus} -ne 0 ];
then
fail "Release prepare failed."
exit ${prepareStatus}
else
pass "Release prepare Success."
fi
# ----------------------------------------------------------------------------------------------------
# Check if build for this tag is reproducible
git checkout "$(git describe --abbrev=0)"
# ----------------------------------------------------------------------------------------------------
info "Publishing for reproduction check to Local repo"
mvn clean install -PpackageForRelease -PskipQuality
reproCheckPublishStatus=$?
if [ ${reproCheckPublishStatus} -ne 0 ];
then
git switch -
fail "Publishing for reproduction check failed."
exit ${reproCheckPublishStatus}
else
pass "Publishing for reproduction check Success."
fi
# ----------------------------------------------------------------------------------------------------
info "Checking build reproducability ... "
mvn clean verify artifact:compare
reproducibleStatus=$?
git switch -
if [ ${reproducibleStatus} -ne 0 ];
then
fail "Build is NOT reproducible."
exit ${reproducibleStatus}
else
pass "Build is reproducible."
fi
# ----------------------------------------------------------------------------------------------------
# Actually run the release: Effectively mvn deploy towards Sonatype
info "Doing release:perform"
mvn release:perform
performStatus=$?
if [ ${performStatus} -ne 0 ];
then
fail "Release perform failed."
exit ${performStatus}
else
pass "Release perform Success."
fi
# ----------------------------------------------------------------------------------------------------
#
# Now check SONATYPE
#
RELEASEVERSION=$(git describe --abbrev=0| sed 's/^v//')
info "Now verify Sonatype to release version ${RELEASEVERSION}"
info "Go to https://central.sonatype.com/publishing/deployments"
warn "Press any key abort or 'c' to continue and update the website"
read -n 1 k <&1
if [[ $k = c ]] ;
then
pass "Release worked, pushing results"
else
die "Aborting, nothing was pushed."
fi
# ----------------------------------------------------------------------------------------------------
warn "Now go and manually push it all"
# ----------------------------------------------------------------------------------------------------
echo "git push"
echo "git push --tags"
# ----------------------------------------------------------------------------------------------------
================================================
FILE: devtools/src/main/resources/checkstyle/checkstyle.xml
================================================
<?xml version="1.0"?>
<!--
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.2//EN"
"http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
<module name="Checker">
<module name="SuppressionFilter">
<property name="file" value="${checkstyle.suppressions.file}"/>
</module>
<!-- Checks that a package.html file exists for each package. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html#PackageHtml -->
<!-- module name="JavadocPackage"/ -->
<!-- Checks whether files end with a new line. -->
<!-- See http://checkstyle.sf.net/config_misc.html#NewlineAtEndOfFile -->
<module name="NewlineAtEndOfFile">
<property name="lineSeparator" value="lf"/>
</module>
<!-- Checks that property files contain the same keys. -->
<!-- See http://checkstyle.sf.net/config_misc.html#Translation -->
<module name="Translation" />
<module name="FileTabCharacter" />
<module name="FileLength">
<property name="fileExtensions" value="!yaml"/>
</module>
<module name="LineLength">
<property name="max" value="150" />
<property name="fileExtensions" value="!yaml"/>
</module>
<module name="SuppressWithPlainTextCommentFilter">
<property name="offCommentFormat" value="CHECKSTYLE\.OFF\: ([\w\|]+)"/>
<property name="onCommentFormat" value="CHECKSTYLE\.ON\: ([\w\|]+)"/>
<property name="checkFormat" value="$1"/>
</module>
<module name="TreeWalker">
<!-- Checks for Javadoc comments. -->
<!-- See http://checkstyle.sf.net/config_javadoc.html -->
<!-- <module name="JavadocType"> <property name="scope" value="public"/> <property name="allowMissingParamTags" value="true"/> </module> <module name="JavadocStyle"/> -->
<!-- Checks for Naming Conventions. -->
<!-- See http://checkstyle.sf.net/config_naming.html -->
<module name="ConstantName" />
<module name="LocalFinalVariableName" />
<module name="LocalVariableName" />
<module name="MemberName" />
<module name="MethodName">
<property name="format" value="^[a-z][a-zA-Z0-9_]*[a-zA-Z0-9]+$"/>
<!--<message key="name.invalidPattern"-->
<!--message="Method name ''{0}'' must match pattern ''{1}''."/>-->
</module>
<module name="PackageName" />
<module name="ParameterName" />
<module name="StaticVariableName" />
<module name="TypeName" />
<!-- Checks for Headers -->
<!-- See http://checkstyle.sf.net/config_header.html -->
<!-- <module name="Header"> -->
<!-- The follow property value demonstrates the ability -->
<!-- to have access to ANT properties. In this case it uses -->
<!-- the ${basedir} property to allow Checkstyle to be run -->
<!-- from any directory within a project. See property -->
<!-- expansion, -->
<!-- http://checkstyle.sf.net/config.html#properties -->
<!-- <property -->
<!-- name="headerFile" -->
<!-- value="${basedir}/java.header"/> -->
<!-- </module> -->
<!-- Following interprets the header file as regular expressions. -->
<!-- <module name="RegexpHeader"/> -->
<!-- Checks for imports -->
<!-- See http://checkstyle.sf.net/config_import.html -->
<module name="IllegalImport" /> <!-- defaults to sun.* packages -->
<module name="AvoidStarImport"/>
<module name="RedundantImport"/>
<module name="UnusedImports"/>
<module name="EmptyStatement"/>
<module name="IllegalInstantiation"/>
<module name="SimplifyBooleanExpression"/>
<module name="SimplifyBooleanReturn"/>
<module name="InterfaceIsType"/>
<module name="ArrayTypeStyle"/>
<!-- Checks for Size Violations. -->
<!-- See http://checkstyle.sf.net/config_sizes.html -->
<module name="MethodLength">
<property name="max" value="500" />
<property name="countEmpty" value="false" />
</module>
<module name="ParameterNumber" />
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="EmptyForIteratorPad" />
<module name="MethodParamPad" />
<module name="NoWhitespaceAfter" />
<module name="NoWhitespaceBefore" />
<module name="ParenPad" />
<module name="TypecastParenPad" />
<module name="WhitespaceAfter">
<property name="tokens" value="COMMA, SEMI" />
</module>
<!-- Alert about trailing whitespace -->
<module name="Regexp">
<property name="format" value="[ \t]+$"/>
<property name="illegalPattern" value="true"/>
<property name="message" value="Trailing whitespace"/>
</module>
<!-- Modifier Checks -->
<!-- See http://checkstyle.sf.net/config_modifiers.html -->
<module name="ModifierOrder" />
<module name="RedundantModifier" />
<!-- Checks for blocks. You know, those {}'s -->
<!-- See http://checkstyle.sf.net/config_blocks.html -->
<module name="AvoidNestedBlocks" />
<module name="EmptyBlock" />
<module name="LeftCurly" />
<module name="NeedBraces" />
<module name="RightCurly">
<property name="option" value="same" />
</module>
<!-- Checks for common coding problems -->
<!-- See http://checkstyle.sf.net/config_coding.html -->
<!--<module name="AvoidInlineConditionals" />-->
<!-- <module name="DoubleCheckedLocking"/> -->
<module name="EmptyStatement" />
<module name="EqualsHashCode" />
<module name="HiddenField">
<property name="ignoreConstructorParameter" value="true" />
</module>
<module name="IllegalInstantiation" />
<module name="InnerAssignment" />
<module name="MissingSwitchDefault" />
<module name="SimplifyBooleanExpression" />
<module name="SimplifyBooleanReturn" />
<!-- Checks for class design -->
<!-- See http://checkstyle.sf.net/config_design.html -->
<module name="FinalClass" />
<module name="HideUtilityClassConstructor" />
<module name="InterfaceIsType" />
<module name="VisibilityModifier">
<property name="packageAllowed" value="true"/>
<property name="protectedAllowed" value="true"/>
</module>
<!-- Miscellaneous other checks. -->
<!-- See http://checkstyle.sf.net/config_misc.html -->
<module name="ArrayTypeStyle" />
<module name="Indentation">
<property name="basicOffset" value="4" />
<property name="braceAdjustment" value="0" />
<property name="caseIndent" value="4" />
<property name="throwsIndent" value="4" />
<property name="arrayInitIndent" value="0" />
<property name="lineWrappingIndentation" value="4" />
</module>
<!-- module name="TodoComment"/ -->
<module name="UpperEll" />
</module>
</module>
================================================
FILE: devtools/src/main/resources/checkstyle/suppressions.xml
================================================
<?xml version="1.0"?>
<!--
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<!DOCTYPE suppressions PUBLIC
"-//Puppy Crawl//DTD Suppressions 1.1//EN"
"http://www.puppycrawl.com/dtds/suppressions_1_1.dtd">
<suppressions>
<!-- Avoid any reports about generated code -->
<suppress files=".*[\\/]target[\\/]" checks=".*" />
<!-- Junit 5 does not want 'public' -->
<!-- Without public this rule enforces removing public from constructors -->
<!-- Causing the tests to fail because now the reflection no longer works. -->
<suppress checks="RedundantModifier" files=".*[\\/]src[\\/](test|it)[\\/]"/>
</suppressions>
================================================
FILE: docs/CNAME
================================================
logparser.basjes.nl
================================================
FILE: docs/README.md
================================================
Apache HTTPD & NGINX access log parser
======================================
[](https://github.com/nielsbasjes/logparser/actions)
[](https://app.codecov.io/gh/nielsbasjes/logparser)
[](https://www.apache.org/licenses/LICENSE-2.0.html)
[](https://search.maven.org/#search%7Cga%7C1%7Cg%3A%22nl.basjes.parse.httpdlog%22)
[](https://www.paypal.me/nielsbasjes)
This is a Logparsing framework intended to make parsing [Apache HTTPD](https://httpd.apache.org/) and [NGINX](https://nginx.org/) access log files much easier.
The basic idea is that you should be able to have a parser that you can construct by simply
telling it with what configuration options the line was written.
These configuration options are the schema of the access loglines.
So we are using the LogFormat that wrote the file as the input parameter for the parser that reads the same file.
In addition to the config options specified in the Apache HTTPD manual under
[Custom Log Formats](https://httpd.apache.org/docs/current/mod/mod_log_config.html) the following are also recognized:
* common
* combined
* combinedio
* referer
* agent
For Nginx the log_format tokens are specified [here](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) and [here](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables).
*** PLACE HOLDER PAGE ***
Donations
===
If this project has business value for you then don't hesitate to support me with a small donation.
[](https://www.paypal.me/nielsbasjes)
License
===
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
================================================
FILE: examples/apache-beam/.gitignore
================================================
tmp
================================================
FILE: examples/apache-beam/pom.xml
================================================
<?xml version="1.0"?>
<!--
Apache HTTPD & NGINX Access log parsing made easy
Copyright (C) 2011-2023 Niels Basjes
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>httpdlog-examples</artifactId>
<groupId>nl.basjes.parse.httpdlog.examples</groupId>
<version>6.0.1-SNAPSHOT</version>
</parent>
<artifactId>apache-beam</artifactId>
<name>Parser - Examples - Apache Beam</name>
<properties>
<!-- The mess of the dependencies is too big to make this check pass -->
<depencency-convergence.phase>none</depencency-convergence.phase>
</properties>
<dependencies>
<dependency>
<groupId>nl.basjes.parse.httpdlog</groupId>
<artifactId>httpdlog-parser</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>${beam.version}</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
<version>${avro.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-sdks-java-core</artifactId>
<version>${beam.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.beam</groupId>
<artifactId>beam-runners-direct-java</artifactId>
<version>${beam.version}</version>
<scope>test</scope>
</dependency>
<!--=========================-->
<!--Needed for PAssert-->
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
<version>1.3</version>
<scope>test</scope>
</dependency>
<!--Needed for PAssert-->
<!--=========================-->
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.jacoco</groupId>
<artifactId>jacoco-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.avro</groupId>
<artifactId>avro-maven-plugin</artifactId>
<executions>
<!--<execution>-->
<!--<id>schemas</id>-->
<!--<phase>generate-sources</phase>-->
<!--<goals>-->
<!--<goal>schema</goal>-->
<!--<goal>protocol</goal>-->
<!--<goal>idl-protocol</goal>-->
<!--</goals>-->
<!--<configuration>-->
<!--<stringType>String</stringType>-->
<!--<sourceDirectory>src/main/avro</sourceDirectory>-->
<!--<fieldVisibility>private</fieldVisibility>-->
<!--</configuration>-->
<!--</execution>-->
<execution>
<id>test-schemas</id>
<phase>generate-test-sources</phase>
<goals>
<goal>schema</goal>
<goal>protocol</goal>
<goal>idl-protocol</goal>
</goals>
<configuration>
<stringType>String</stringType>
<sourceDirectory>src/test/avro</sourceDirectory>
<fieldVisibility>private</fieldVisibility>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
================================================
FILE: examples/apache-beam/src/test/avro/Record.avdl
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
@namespace("nl.basjes.parse.webevents")
protocol BeamTestRecord {
record Device {
long screenWidth ; // "SCREENWIDTH:request.firstline.uri.query.s.width"
long screenHeight ; // "SCREENHEIGHT:request.firstline.uri.query.s.height"
}
record Browser {
string useragent ; // "STRING:request.user-agent"
}
record ISP {
string asnNumber ; // ASN:connection.client.host.asn.number
string asnOrganization ; // STRING:connection.client.host.asn.organization
string ispName ; // STRING:connection.client.host.isp.name
string ispOrganization ; // STRING:connection.client.host.isp.organization
}
record GeoLocation {
string continentName; // STRING:connection.client.host.continent.name
string continentCode; // STRING:connection.client.host.continent.code
string countryName; // STRING:connection.client.host.country.name
string countryIso; // STRING:connection.client.host.country.iso
string subdivisionName; // STRING:connection.client.host.subdivision.name
string subdivisionIso; // STRING:connection.client.host.subdivision.iso
string cityName; // STRING:connection.client.host.city.name
string postalCode; // STRING:connection.client.host.postal.code
double locationLatitude; // STRING:connection.client.host.location.latitude
double locationLongitude; // STRING:connection.client.host.location.longitude
}
record Visitor {
string ip; // "IP:connection.client.host"
ISP isp;
GeoLocation geoLocation;
}
record Click {
long timestamp; // "TIME.EPOCH:request.receive.time.epoch"
Device device;
Browser browser;
Visitor visitor;
}
}
================================================
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/TestCase.java
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.basjes.parse.httpdlog.beam;
import nl.basjes.parse.core.Parser;
import nl.basjes.parse.httpdlog.HttpdLoglineParser;
import nl.basjes.parse.httpdlog.beam.pojo.MyRecord;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPCityDissector;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPISPDissector;
// CHECKSTYLE.OFF: LineLength
// CHECKSTYLE.OFF: LeftCurly
// CHECKSTYLE.OFF: HideUtilityClassConstructor
public final class TestCase {
private static final String TEST_MMDB_BASE_DIR = "../../GeoIP2-TestData/test-data/";
public static final String ISP_TEST_MMDB = TEST_MMDB_BASE_DIR + "GeoIP2-ISP-Test.mmdb";
public static final String CITY_TEST_MMDB = TEST_MMDB_BASE_DIR + "GeoIP2-City-Test.mmdb";
public static String getLogFormat() {
return "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\"";
}
public static String getInputLine() {
return "2001:980:91c0:1:8d31:a232:25e5:85d - - [05/Sep/2010:11:27:50 +0200] \"GET /b/ss/advbolprod2/1/H.22.1/s73176445413647?AQB=1&pccr=true&vidn=27F07A1B85012045-4000011500517C43&&ndh=1&t=19%2F5%2F2012%2023%3A51%3A27%202%20-120&ce=UTF-8&ns=bol&pageName=%2Fnl%2Fp%2Ffissler-speciaal-pannen-grillpan-28-x-28-cm%2F9200000002876066%2F&g=http%3A%2F%2Fwww.bol.com%2Fnl%2Fp%2Ffissler-speciaal-pannen-grillpan-28-x-28-cm%2F9200000002876066%2F%3Fpromo%3Dkoken-pannen_303_hs-koken-pannen-afj-120601_B3_product_1_9200000002876066%26bltg.pg_nm%3Dkoken-pannen%26bltg.slt_id%3D303%26bltg.slt_nm%3Dhs-koken-pannen-afj-120601%26bltg.slt_p&r=http%3A%2F%2Fwww.bol.com%2Fnl%2Fm%2Fkoken-tafelen%2Fkoken-pannen%2FN%2F11766%2Findex.html%3Fblabla%3Dblablawashere&cc=EUR&ch=D%3Dv3&server=ps316&events=prodView%2Cevent1%2Cevent2%2Cevent31&products=%3B9200000002876066%3B%3B%3B%3Bevar3%3Dkth%7Cevar8%3D9200000002876066_Fissler%20Speciaal%20Pannen%20-%20Grillpan%20-%2028%20x%2028%20cm%7Cevar35%3D170%7Cevar47%3DKTH%7Cevar9%3DNew%7Cevar40%3Dno%20reviews%2C%3B%3B%3B%3Bevent31%3D423&c1=catalog%3Akth%3Aproduct-detail&v1=D%3Dc1&h1=catalog%2Fkth%2Fproduct-detail&h2=D%3DpageName&v3=kth&l3=endeca_001-mensen_default%2Cendeca_exact-boeken_default%2Cendeca_verschijningsjaar_default%2Cendeca_hardgoodscategoriesyn_default%2Cendeca_searchrank-hadoop_default%2Cendeca_genre_default%2Cendeca_uitvoering_default&v4=ps316&v6=koken-pannen_303_hs-koken-pannen-afj-120601_B3_product_1_9200000002876066&v10=Tu%2023%3A30&v12=logged%20in&v13=New&c25=niet%20ssl&c26=3631&c30=84.106.227.113.1323208998208762&v31=2000285551&c45=20120619235127&c46=20120501%204.3.4.1&c47=D%3Ds_vi&c49=%2Fnl%2Fcatalog%2Fproduct-detail.jsp&c50=%2Fnl%2Fcatalog%2Fproduct-detail.jsp&v51=www.bol.com&s=1280x800&c=24&j=1.7&v=N&k=Y&bw=1280&bh=272&p=Shockwave%20Flash%3B&AQE=1 HTTP/1.1\" 200 23617 \"http://www.google.nl/imgres?imgurl=http://daniel_en_sander.basjes.nl/fotos/geboorte-kaartje/geboortekaartje-binnenkant.jpg&imgrefurl=http://daniel_en_sander.basjes.nl/fotos/geboorte-kaartje&usg=__LDxRMkacRs6yLluLcIrwoFsXY6o=&h=521&w=1024&sz=41&hl=nl&start=13&zoom=1&um=1&itbs=1&tbnid=Sqml3uGbjoyBYM:&tbnh=76&tbnw=150&prev=/images%3Fq%3Dbinnenkant%2Bgeboortekaartje%26um%3D1%26hl%3Dnl%26sa%3DN%26biw%3D1882%26bih%3D1014%26tbs%3Disch:1\" \"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; nl-nl) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8\" \"jquery-ui-theme=Eggplant; BuI=SomeThing; Apache=127.0.0.1.1351111543699529\"";
}
public static Parser<MyRecord> createTestParser() throws NoSuchMethodException {
Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, getLogFormat());
parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector());
parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");
parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
parser.addParseTarget("setRequestReceiveTime", "TIME.STAMP:request.receive.time");
parser.addParseTarget("setReferrer", "STRING:request.firstline.uri.query.g.query.promo");
parser.addParseTarget("setScreenResolution", "STRING:request.firstline.uri.query.s");
parser.addParseTarget("setScreenWidth", "SCREENWIDTH:request.firstline.uri.query.s.width");
parser.addParseTarget("setScreenHeight", "SCREENHEIGHT:request.firstline.uri.query.s.height");
parser.addParseTarget("setGoogleQuery", "STRING:request.firstline.uri.query.r.query.blabla");
parser.addParseTarget("setBui", "HTTP.COOKIE:request.cookies.bui");
parser.addParseTarget("setUseragent", "HTTP.USERAGENT:request.user-agent");
parser.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB));
parser.addParseTarget("setAsnNumber", "ASN:connection.client.host.asn.number");
parser.addParseTarget("setAsnOrganization", "STRING:connection.client.host.asn.organization");
parser.addParseTarget("setIspName", "STRING:connection.client.host.isp.name");
parser.addParseTarget("setIspOrganization", "STRING:connection.client.host.isp.organization");
parser.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
parser.addParseTarget("setContinentName", "STRING:connection.client.host.continent.name");
parser.addParseTarget("setContinentCode", "STRING:connection.client.host.continent.code");
parser.addParseTarget("setCountryName", "STRING:connection.client.host.country.name");
parser.addParseTarget("setCountryIso", "STRING:connection.client.host.country.iso");
parser.addParseTarget("setSubdivisionName", "STRING:connection.client.host.subdivision.name");
parser.addParseTarget("setSubdivisionIso", "STRING:connection.client.host.subdivision.iso");
parser.addParseTarget("setCityName", "STRING:connection.client.host.city.name");
parser.addParseTarget("setPostalCode", "STRING:connection.client.host.postal.code");
parser.addParseTarget("setLocationLatitude", "STRING:connection.client.host.location.latitude");
parser.addParseTarget("setLocationLongitude", "STRING:connection.client.host.location.longitude");
return parser;
}
public static String getExpectedConnectionClientHost() { return "2001:980:91c0:1:8d31:a232:25e5:85d"; }
public static String getExpectedRequestReceiveTime() { return "05/Sep/2010:11:27:50 +0200"; }
public static Long getExpectedRequestReceiveTimeEpoch() { return 1283678870000L; }
public static String getExpectedReferrer() { return "koken-pannen_303_hs-koken-pannen-afj-120601_B3_product_1_9200000002876066"; }
public static String getExpectedScreenResolution() { return "1280x800"; }
public static Long getExpectedScreenWidth() { return 1280L; }
public static Long getExpectedScreenHeight() { return 800L; }
public static String getExpectedGoogleQuery() { return "blablawashere"; }
public static String getExpectedBui() { return "SomeThing"; }
public static String getExpectedUseragent() { return "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; nl-nl) AppleWebKit/533.17.8 (KHTML, like Gecko) Version/5.0.1 Safari/533.17.8"; }
public static String getExpectedAsnNumber() { return "6666"; }
public static String getExpectedAsnOrganization() { return "Basjes Global Network IPv6"; }
public static String getExpectedIspName() { return "Basjes ISP IPv6"; }
public static String getExpectedIspOrganization() { return "Niels Basjes IPv6"; }
public static String getExpectedContinentName() { return "Europe"; }
public static String getExpectedContinentCode() { return "EU"; }
public static String getExpectedCountryName() { return "Netherlands"; }
public static String getExpectedCountryIso() { return "NL"; }
public static String getExpectedSubdivisionName() { return "Noord Holland"; }
public static String getExpectedSubdivisionIso() { return "NH"; }
public static String getExpectedCityName() { return "Amstelveen"; }
public static String getExpectedPostalCode() { return "1187"; }
public static Double getExpectedLocationLatitude() { return 52.5; }
public static Double getExpectedLocationLongitude() { return 5.75; }
}
================================================
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/ExpectedClick.java
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.basjes.parse.httpdlog.beam.avro;
import nl.basjes.parse.webevents.Click;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedAsnNumber;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedAsnOrganization;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCityName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedConnectionClientHost;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedContinentCode;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedContinentName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCountryIso;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCountryName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedIspName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedIspOrganization;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedLocationLatitude;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedLocationLongitude;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedPostalCode;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedRequestReceiveTimeEpoch;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedScreenHeight;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedScreenWidth;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedSubdivisionIso;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedSubdivisionName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedUseragent;
// CHECKSTYLE.OFF: HideUtilityClassConstructor
public class ExpectedClick {
public static Click create(){
Click.Builder builder = Click.newBuilder();
builder
.setTimestamp(getExpectedRequestReceiveTimeEpoch())
.getDeviceBuilder()
.setScreenWidth(getExpectedScreenWidth())
.setScreenHeight(getExpectedScreenHeight());
builder
.getBrowserBuilder()
.setUseragent(getExpectedUseragent());
builder
.getVisitorBuilder()
.setIp(getExpectedConnectionClientHost());
builder
.getVisitorBuilder()
.getIspBuilder()
.setAsnNumber(getExpectedAsnNumber())
.setAsnOrganization(getExpectedAsnOrganization())
.setIspName(getExpectedIspName())
.setIspOrganization(getExpectedIspOrganization());
builder
.getVisitorBuilder()
.getGeoLocationBuilder()
.setContinentName(getExpectedContinentName())
.setContinentCode(getExpectedContinentCode())
.setCountryName(getExpectedCountryName())
.setCountryIso(getExpectedCountryIso())
.setSubdivisionName(getExpectedSubdivisionName())
.setSubdivisionIso(getExpectedSubdivisionIso())
.setCityName(getExpectedCityName())
.setPostalCode(getExpectedPostalCode())
.setLocationLatitude(getExpectedLocationLatitude())
.setLocationLongitude(getExpectedLocationLongitude());
return builder.build();
}
}
================================================
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvro.java
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.basjes.parse.httpdlog.beam.avro;
import nl.basjes.parse.core.Field;
import nl.basjes.parse.core.Parser;
import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.core.exceptions.InvalidDissectorException;
import nl.basjes.parse.core.exceptions.MissingDissectorsException;
import nl.basjes.parse.httpdlog.HttpdLoglineParser;
import nl.basjes.parse.httpdlog.beam.TestCase;
import nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPCityDissector;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPISPDissector;
import nl.basjes.parse.webevents.Click;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.commons.lang3.builder.Builder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import static nl.basjes.parse.httpdlog.beam.TestCase.CITY_TEST_MMDB;
import static nl.basjes.parse.httpdlog.beam.TestCase.ISP_TEST_MMDB;
// CHECKSTYLE.OFF: LineLength
// CHECKSTYLE.OFF: LeftCurly
@RunWith(JUnit4.class)
public class TestParserDoFnAvro implements Serializable {
public static class ClickSetter implements Builder<Click> {
final Click.Builder builder = Click.newBuilder();
@Field("TIME.EPOCH:request.receive.time.epoch") public void setRequestReceiveTime(Long value) { builder.setTimestamp(value); }
@Field("SCREENWIDTH:request.firstline.uri.query.s.width") public void setScreenWidth(Long value) { builder.getDeviceBuilder().setScreenWidth(value); }
@Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public void setScreenHeight(Long value) { builder.getDeviceBuilder().setScreenHeight(value); }
@Field("HTTP.USERAGENT:request.user-agent") public void setUseragent(String value) { builder.getBrowserBuilder().setUseragent(value); }
@Field("IP:connection.client.host") public void setConnectionClientHost(String value) { builder.getVisitorBuilder().setIp(value); }
@Field("ASN:connection.client.host.asn.number") public void setAsnNumber(String value) { builder.getVisitorBuilder().getIspBuilder().setAsnNumber(value); }
@Field("STRING:connection.client.host.asn.organization") public void setAsnOrganization(String value) { builder.getVisitorBuilder().getIspBuilder().setAsnOrganization(value); }
@Field("STRING:connection.client.host.isp.name") public void setIspName(String value) { builder.getVisitorBuilder().getIspBuilder().setIspName(value); }
@Field("STRING:connection.client.host.isp.organization") public void setIspOrganization(String value) { builder.getVisitorBuilder().getIspBuilder().setIspOrganization(value); }
@Field("STRING:connection.client.host.continent.name") public void setContinentName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setContinentName(value); }
@Field("STRING:connection.client.host.continent.code") public void setContinentCode(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setContinentCode(value); }
@Field("STRING:connection.client.host.country.name") public void setCountryName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCountryName(value); }
@Field("STRING:connection.client.host.country.iso") public void setCountryIso(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCountryIso(value); }
@Field("STRING:connection.client.host.subdivision.name") public void setSubdivisionName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setSubdivisionName(value); }
@Field("STRING:connection.client.host.subdivision.iso") public void setSubdivisionIso(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setSubdivisionIso(value); }
@Field("STRING:connection.client.host.city.name") public void setCityName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCityName(value); }
@Field("STRING:connection.client.host.postal.code") public void setPostalCode(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setPostalCode(value); }
@Field("STRING:connection.client.host.location.latitude") public void setLocationLatitude(Double value) { builder.getVisitorBuilder().getGeoLocationBuilder().setLocationLatitude(value); }
@Field("STRING:connection.client.host.location.longitude") public void setLocationLongitude(Double value) { builder.getVisitorBuilder().getGeoLocationBuilder().setLocationLongitude(value); }
@Override
public Click build() {
return builder.build();
}
}
public static class MyParserDoFn extends DoFn<String, Click> {
private Parser<ClickSetter> parser;
@Setup
public void setup() {
parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat())
.addDissector(new ScreenResolutionDissector())
.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION")
.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB))
.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
}
@ProcessElement
public void processElement(ProcessContext c) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
String input = c.element();
ClickSetter setter = parser.parse(new ClickSetter(), input);
if (setter == null) {
System.err.println("Something went terribly wrong");
return;
}
Click click = setter.build();
c.output(click);
}
}
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
@Test
public void testClassDefinitionAvro() {
List<String> logLines = Collections.singletonList(TestCase.getInputLine());
// Apply Create, passing the list and the coder, to create the PCollection.
PCollection<String> input = pipeline.apply(Create.of(logLines)).setCoder(StringUtf8Coder.of());
PCollection<Click> filledTestRecords = input
.apply("Extract Elements from logline",
ParDo.of(new MyParserDoFn()));
PAssert.that(filledTestRecords).containsInAnyOrder(ExpectedClick.create());
pipeline.run().waitUntilFinish();
}
}
================================================
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvroInline.java
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.basjes.parse.httpdlog.beam.avro;
import nl.basjes.parse.core.Field;
import nl.basjes.parse.core.Parser;
import nl.basjes.parse.core.exceptions.DissectionFailure;
import nl.basjes.parse.core.exceptions.InvalidDissectorException;
import nl.basjes.parse.core.exceptions.MissingDissectorsException;
import nl.basjes.parse.httpdlog.HttpdLoglineParser;
import nl.basjes.parse.httpdlog.beam.TestCase;
import nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPCityDissector;
import nl.basjes.parse.httpdlog.dissectors.geoip.GeoIPISPDissector;
import nl.basjes.parse.webevents.Click;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.TestPipeline;
import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.values.PCollection;
import org.apache.commons.lang3.builder.Builder;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import java.io.Serializable;
import java.util.Collections;
import java.util.List;
import static nl.basjes.parse.httpdlog.beam.TestCase.CITY_TEST_MMDB;
import static nl.basjes.parse.httpdlog.beam.TestCase.ISP_TEST_MMDB;
// CHECKSTYLE.OFF: LineLength
// CHECKSTYLE.OFF: LeftCurly
@RunWith(JUnit4.class)
public class TestParserDoFnAvroInline implements Serializable {
public static class ClickSetter implements Builder<Click> {
final Click.Builder builder = Click.newBuilder();
@Field("TIME.EPOCH:request.receive.time.epoch") public void setRequestReceiveTime(Long value) { builder.setTimestamp(value); }
@Field("SCREENWIDTH:request.firstline.uri.query.s.width") public void setScreenWidth(Long value) { builder.getDeviceBuilder().setScreenWidth(value); }
@Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public void setScreenHeight(Long value) { builder.getDeviceBuilder().setScreenHeight(value); }
@Field("HTTP.USERAGENT:request.user-agent") public void setUseragent(String value) { builder.getBrowserBuilder().setUseragent(value); }
@Field("IP:connection.client.host") public void setConnectionClientHost(String value) { builder.getVisitorBuilder().setIp(value); }
@Field("ASN:connection.client.host.asn.number") public void setAsnNumber(String value) { builder.getVisitorBuilder().getIspBuilder().setAsnNumber(value); }
@Field("STRING:connection.client.host.asn.organization") public void setAsnOrganization(String value) { builder.getVisitorBuilder().getIspBuilder().setAsnOrganization(value); }
@Field("STRING:connection.client.host.isp.name") public void setIspName(String value) { builder.getVisitorBuilder().getIspBuilder().setIspName(value); }
@Field("STRING:connection.client.host.isp.organization") public void setIspOrganization(String value) { builder.getVisitorBuilder().getIspBuilder().setIspOrganization(value); }
@Field("STRING:connection.client.host.continent.name") public void setContinentName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setContinentName(value); }
@Field("STRING:connection.client.host.continent.code") public void setContinentCode(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setContinentCode(value); }
@Field("STRING:connection.client.host.country.name") public void setCountryName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCountryName(value); }
@Field("STRING:connection.client.host.country.iso") public void setCountryIso(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCountryIso(value); }
@Field("STRING:connection.client.host.subdivision.name") public void setSubdivisionName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setSubdivisionName(value); }
@Field("STRING:connection.client.host.subdivision.iso") public void setSubdivisionIso(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setSubdivisionIso(value); }
@Field("STRING:connection.client.host.city.name") public void setCityName(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setCityName(value); }
@Field("STRING:connection.client.host.postal.code") public void setPostalCode(String value) { builder.getVisitorBuilder().getGeoLocationBuilder().setPostalCode(value); }
@Field("STRING:connection.client.host.location.latitude") public void setLocationLatitude(Double value) { builder.getVisitorBuilder().getGeoLocationBuilder().setLocationLatitude(value); }
@Field("STRING:connection.client.host.location.longitude") public void setLocationLongitude(Double value) { builder.getVisitorBuilder().getGeoLocationBuilder().setLocationLongitude(value); }
@Override
public Click build() {
return builder.build();
}
}
@Rule
public final transient TestPipeline pipeline = TestPipeline.create();
@Test
public void testClassDefinitionAvro() {
List<String> logLines = Collections.singletonList(TestCase.getInputLine());
// Apply Create, passing the list and the coder, to create the PCollection.
PCollection<String> input = pipeline.apply(Create.of(logLines)).setCoder(StringUtf8Coder.of());
PCollection<Click> filledTestRecords = input
.apply("Extract Elements from logline",
ParDo.of(new DoFn<String, Click>() {
private Parser<ClickSetter> parser;
@Setup
public void setup() {
parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat())
.addDissector(new ScreenResolutionDissector())
.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI")
.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION")
.addDissector(new GeoIPISPDissector(ISP_TEST_MMDB))
.addDissector(new GeoIPCityDissector(CITY_TEST_MMDB));
}
@ProcessElement
public void processElement(ProcessContext c) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
String input = c.element();
ClickSetter setter = parser.parse(new ClickSetter(), input);
Click click = setter.build();
c.output(click);
}
}));
PAssert.that(filledTestRecords).containsInAnyOrder(ExpectedClick.create());
pipeline.run().waitUntilFinish();
}
}
================================================
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/MyRecord.java
================================================
/*
* Apache HTTPD & NGINX Access log parsing made easy
* Copyright (C) 2011-2023 Niels Basjes
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package nl.basjes.parse.httpdlog.beam.pojo;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Setter;
import lombok.ToString;
import org.junit.Test;
import java.io.Serializable;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedAsnNumber;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedAsnOrganization;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedBui;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCityName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedConnectionClientHost;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedContinentCode;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedContinentName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCountryIso;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedCountryName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedGoogleQuery;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedIspName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedIspOrganization;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedLocationLatitude;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedLocationLongitude;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedPostalCode;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedReferrer;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedRequestReceiveTime;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedScreenHeight;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedScreenResolution;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedScreenWidth;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedSubdivisionIso;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedSubdivisionName;
import static nl.basjes.parse.httpdlog.beam.TestCase.getExpectedUseragent;
import static org.junit.Assert.assertEquals;
// CHECKSTYLE.OFF: ParamPad
@ToString
@EqualsAndHashCode
public class MyRecord implements Serializable {
@Getter @Setter private String connectionClientHost = null;
@Getter @Setter private String requestReceiveTime = null;
@Getter @Setter private String referrer = null;
@Getter @Setter private String screenResolution = null;
@Getter @Setter private Long screenWidth = null;
@Getter @Setter private Long screenHeight = null;
@Getter @Setter private String googleQuery = null;
@Getter @Setter private String bui = null;
@Getter @Setter private String useragent = null;
@Getter @Setter private String asnNumber = null;
@Getter @Setter private String asnOrganization = null;
@Getter @Setter private String ispName = null;
@Getter @Setter private String ispOrganization = null;
@Getter @Setter private String continentName = null;
@Getter @Setter private String continentCode = null;
@Getter @Setter private String countryName = null;
@Getter @Setter private String countryIso = null;
@Getter @Setter private String subdivisionName = null;
@Getter @Setter private String subdivisionIso = null;
@Getter @Setter private String cityName = null;
@Getter @Setter private String postalCode = null;
@Getter @Setter private Double locationLatitude = null;
@Getter @Setter private Double locationLongitude = null;
public void assertIsValid() {
assertEquals(getExpectedConnectionClientHost(), getConnectionClientHost());
assertEquals(getExpectedRequestReceiveTime(), getRequestReceiveTime());
assertEquals(getExpectedReferrer(), getReferrer());
assertEquals(getExpectedScreenResolution(), getScreenResolution());
assertEquals(getExpectedScreenWidth(), getScreenWidth());
assertEquals(getExpectedScreenHeight(), getScreenHeight());
assertEquals(getExpectedGoogleQuery(), getGoogleQuery());
assertEquals(getExpectedBui(), getBui());
assertEquals(getEx
gitextract_a39vjp3u/
├── .editorconfig
├── .github/
│ ├── FUNDING.yml
│ └── workflows/
│ └── build.yml
├── .gitignore
├── .pmd
├── .travis.yml__
├── CHANGELOG.md
├── CNAME
├── GeoIP2-TestData/
│ ├── Dockerfile
│ ├── README.md
│ ├── rebuild.sh
│ ├── source-data/
│ │ ├── GeoIP2-City-Test.json
│ │ ├── GeoIP2-Country-Test.json
│ │ ├── GeoIP2-ISP-Test.json
│ │ └── GeoLite2-ASN-Test.json
│ └── test-data/
│ ├── GeoIP2-City-Test.mmdb
│ ├── GeoIP2-Country-Test.mmdb
│ ├── GeoIP2-ISP-Test.mmdb
│ ├── GeoLite2-ASN-Test.mmdb
│ └── write-test-data.pl
├── LICENSE
├── README-Hive.md
├── README-Java.md
├── README-Pig.md
├── README-geoip.md
├── README.md
├── _config.yml
├── devtools/
│ ├── docker/
│ │ ├── Dockerfile
│ │ ├── bashcolors.sh
│ │ ├── build_env_checks.sh
│ │ ├── configure-for-user.sh
│ │ ├── env.sh
│ │ └── prompt.sh
│ ├── logformat.conf
│ ├── pom.xml
│ ├── release.sh
│ └── src/
│ └── main/
│ └── resources/
│ └── checkstyle/
│ ├── checkstyle.xml
│ └── suppressions.xml
├── docs/
│ ├── CNAME
│ └── README.md
├── examples/
│ ├── apache-beam/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── avro/
│ │ │ └── Record.avdl
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── beam/
│ │ │ ├── TestCase.java
│ │ │ ├── avro/
│ │ │ │ ├── ExpectedClick.java
│ │ │ │ ├── TestParserDoFnAvro.java
│ │ │ │ └── TestParserDoFnAvroInline.java
│ │ │ └── pojo/
│ │ │ ├── MyRecord.java
│ │ │ ├── TestParserDoFnClass.java
│ │ │ └── TestParserDoFnInline.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── apache-flink/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── test/
│ │ ├── avro/
│ │ │ └── Record.avdl
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── flink/
│ │ │ ├── TestCase.java
│ │ │ ├── avro/
│ │ │ │ ├── ExpectedClick.java
│ │ │ │ ├── TestParserMapFunctionAvroClass.java
│ │ │ │ └── TestParserMapFunctionAvroInline.java
│ │ │ └── pojo/
│ │ │ ├── MyRecord.java
│ │ │ ├── TestParserMapFunctionClass.java
│ │ │ └── TestParserMapFunctionInline.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── apache-hadoop-mapreduce/
│ │ ├── .gitignore
│ │ ├── pom.xml
│ │ └── src/
│ │ └── main/
│ │ ├── assembly/
│ │ │ └── job.xml
│ │ └── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── hadoop/
│ │ └── io/
│ │ └── input/
│ │ └── Wordcount.java
│ ├── demolog/
│ │ ├── README.md
│ │ └── hackers-access.log
│ ├── java-pojo/
│ │ ├── pom.xml
│ │ └── src/
│ │ └── main/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ ├── Main.java
│ │ │ └── MyRecord.java
│ │ └── resources/
│ │ └── log4j.properties
│ └── pom.xml
├── httpdlog/
│ ├── httpdlog-inputformat/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── assembly/
│ │ │ │ └── job.xml
│ │ │ └── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── hadoop/
│ │ │ └── input/
│ │ │ ├── ApacheHttpdLogfileInputFormat.java
│ │ │ ├── ApacheHttpdLogfileRecordReader.java
│ │ │ └── ParsedRecord.java
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── hadoop/
│ │ │ └── input/
│ │ │ ├── TestApacheHttpdLogfileInputFormat.java
│ │ │ ├── TestGetAllFields.java
│ │ │ └── TestParsedRecord.java
│ │ └── resources/
│ │ ├── access.log
│ │ └── log4j.properties
│ ├── httpdlog-parser/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── antlr4/
│ │ │ │ └── nl/
│ │ │ │ └── basjes/
│ │ │ │ └── parse/
│ │ │ │ └── strftime/
│ │ │ │ └── StrfTime.g4
│ │ │ ├── assembly/
│ │ │ │ └── job.xml
│ │ │ ├── java/
│ │ │ │ └── nl/
│ │ │ │ └── basjes/
│ │ │ │ └── parse/
│ │ │ │ └── httpdlog/
│ │ │ │ ├── ApacheHttpdLogFormatDissector.java
│ │ │ │ ├── HttpdLogFormatDissector.java
│ │ │ │ ├── HttpdLoglineParser.java
│ │ │ │ ├── NginxHttpdLogFormatDissector.java
│ │ │ │ ├── Utils.java
│ │ │ │ └── dissectors/
│ │ │ │ ├── HttpFirstLineDissector.java
│ │ │ │ ├── HttpFirstLineProtocolDissector.java
│ │ │ │ ├── HttpUriDissector.java
│ │ │ │ ├── ModUniqueIdDissector.java
│ │ │ │ ├── QueryStringFieldDissector.java
│ │ │ │ ├── RequestCookieListDissector.java
│ │ │ │ ├── ResponseSetCookieDissector.java
│ │ │ │ ├── ResponseSetCookieListDissector.java
│ │ │ │ ├── ScreenResolutionDissector.java
│ │ │ │ ├── StrfTimeStampDissector.java
│ │ │ │ ├── StrfTimeToDateTimeFormatter.java
│ │ │ │ ├── TimeStampDissector.java
│ │ │ │ ├── geoip/
│ │ │ │ │ ├── AbstractGeoIPDissector.java
│ │ │ │ │ ├── GeoIPASNDissector.java
│ │ │ │ │ ├── GeoIPCityDissector.java
│ │ │ │ │ ├── GeoIPCountryDissector.java
│ │ │ │ │ └── GeoIPISPDissector.java
│ │ │ │ ├── nginxmodules/
│ │ │ │ │ ├── CoreLogModule.java
│ │ │ │ │ ├── GeoIPModule.java
│ │ │ │ │ ├── KubernetesIngressModule.java
│ │ │ │ │ ├── NginxModule.java
│ │ │ │ │ ├── SslModule.java
│ │ │ │ │ ├── UpstreamListDissector.java
│ │ │ │ │ ├── UpstreamModule.java
│ │ │ │ │ └── VariousModule.java
│ │ │ │ ├── tokenformat/
│ │ │ │ │ ├── NamedTokenParser.java
│ │ │ │ │ ├── ParameterizedTokenParser.java
│ │ │ │ │ ├── Token.java
│ │ │ │ │ ├── TokenFormatDissector.java
│ │ │ │ │ ├── TokenOutputField.java
│ │ │ │ │ ├── TokenParser.java
│ │ │ │ │ └── TokenSorterByStartPos.java
│ │ │ │ └── translate/
│ │ │ │ ├── ConvertCLFIntoNumber.java
│ │ │ │ ├── ConvertMillisecondsIntoMicroseconds.java
│ │ │ │ ├── ConvertNumberIntoCLF.java
│ │ │ │ ├── ConvertSecondsWithMillisStringDissector.java
│ │ │ │ └── TypeConvertBaseDissector.java
│ │ │ └── resources/
│ │ │ └── version/
│ │ │ └── Version.java.template
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ ├── ApacheHttpdAllFieldsTest.java
│ │ │ ├── ApacheHttpdLogParserTest.java
│ │ │ ├── BasicOverallTest.java
│ │ │ ├── ClientHintsTest.java
│ │ │ ├── CookiesTest.java
│ │ │ ├── EdgeCasesTest.java
│ │ │ ├── JettyLogFormatParserTest.java
│ │ │ ├── JsonLogFormatTest.java
│ │ │ ├── MultiLineHttpdLogParserTest.java
│ │ │ ├── NginxLogFormatJsonTest.java
│ │ │ ├── NginxLogFormatTest.java
│ │ │ ├── UtilsTest.java
│ │ │ ├── dissectors/
│ │ │ │ ├── TestCookieDissector.java
│ │ │ │ ├── TestGeoIPDissectors.java
│ │ │ │ ├── TestHttpFirstLineDissector.java
│ │ │ │ ├── TestHttpUriDissector.java
│ │ │ │ ├── TestModUniqueIdDissector.java
│ │ │ │ ├── TestQueryStringDissector.java
│ │ │ │ └── TestTimeStampDissector.java
│ │ │ ├── nginxmodules/
│ │ │ │ ├── NginxAllFieldsTest.java
│ │ │ │ └── NginxUpstreamTest.java
│ │ │ └── translate/
│ │ │ └── TestTranslators.java
│ │ └── resources/
│ │ └── log4j.properties
│ ├── httpdlog-serde/
│ │ ├── pom.xml
│ │ └── src/
│ │ ├── main/
│ │ │ ├── assembly/
│ │ │ │ └── udf.xml
│ │ │ └── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ └── ApacheHttpdlogDeserializer.java
│ │ └── test/
│ │ ├── java/
│ │ │ └── nl/
│ │ │ └── basjes/
│ │ │ └── parse/
│ │ │ └── httpdlog/
│ │ │ ├── TestAllDissectorTypes.java
│ │ │ └── TestApacheHttpdlogDeserializer.java
│ │ └── resources/
│ │ └── log4j.properties
│ └── pom.xml
├── parser-core/
│ ├── pom.xml
│ └── src/
│ ├── main/
│ │ ├── assembly/
│ │ │ └── job.xml
│ │ └── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── parse/
│ │ └── core/
│ │ ├── Casts.java
│ │ ├── Dissector.java
│ │ ├── Field.java
│ │ ├── Parsable.java
│ │ ├── ParsedField.java
│ │ ├── Parser.java
│ │ ├── SimpleDissector.java
│ │ ├── Value.java
│ │ └── exceptions/
│ │ ├── DissectionFailure.java
│ │ ├── FatalErrorDuringCallOfSetterMethod.java
│ │ ├── InvalidDissectorException.java
│ │ ├── InvalidFieldMethodSignature.java
│ │ └── MissingDissectorsException.java
│ └── test/
│ ├── java/
│ │ └── nl/
│ │ └── basjes/
│ │ └── parse/
│ │ └── core/
│ │ ├── ParserCastsTest.java
│ │ ├── ParserDissectionOutputTypesTest.java
│ │ ├── ParserDuplicateOutputTest.java
│ │ ├── ParserExceptionsTest.java
│ │ ├── ParserInfiniteLoopTest.java
│ │ ├── ParserNormalTest.java
│ │ ├── ParserNormalTestRecord.java
│ │ ├── ParserResetTest.java
│ │ ├── ParserTypeColissionTest.java
│ │ ├── ParserTypeRemappingEdgeCase.java
│ │ ├── TestBadAPIUsage.java
│ │ ├── annotation/
│ │ │ ├── TestFieldSetters.java
│ │ │ ├── TestFieldSettersAlwaysCombined.java
│ │ │ ├── TestFieldSettersAlwaysSeparate.java
│ │ │ ├── TestFieldSettersNotEmpty.java
│ │ │ └── TestFieldSettersNotNull.java
│ │ ├── convert/
│ │ │ └── ValueConvertTest.java
│ │ ├── reference/
│ │ │ ├── BarDissector.java
│ │ │ ├── FooDissector.java
│ │ │ ├── FooSpecialDissector.java
│ │ │ ├── ReferenceTest.java
│ │ │ └── ReferenceTestDouble.java
│ │ └── test/
│ │ ├── DissectorTester.java
│ │ ├── EmptyValuesDissector.java
│ │ ├── MyDissectorTester.java
│ │ ├── NormalValuesDissector.java
│ │ ├── NullValuesDissector.java
│ │ ├── TestRecord.java
│ │ ├── TestUltimateDummyDissector.java
│ │ ├── TestUltimateDummyDissectorFailurelogging.java
│ │ └── UltimateDummyDissector.java
│ └── resources/
│ └── log4j.properties
├── pom.xml
├── renovate.json
├── start-docker.sh
└── utils/
├── PojoGenerator/
│ ├── pom.xml
│ └── src/
│ └── main/
│ └── java/
│ └── nl/
│ └── basjes/
│ └── parse/
│ └── httpdlog/
│ └── PojoGenerator.java
└── pom.xml
SYMBOL INDEX (1511 symbols across 134 files)
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/TestCase.java
class TestCase (line 29) | public final class TestCase {
method getLogFormat (line 35) | public static String getLogFormat() {
method getInputLine (line 39) | public static String getInputLine() {
method createTestParser (line 43) | public static Parser<MyRecord> createTestParser() throws NoSuchMethodE...
method getExpectedConnectionClientHost (line 83) | public static String getExpectedConnectionClientHost() { return "...
method getExpectedRequestReceiveTime (line 84) | public static String getExpectedRequestReceiveTime() { return "...
method getExpectedRequestReceiveTimeEpoch (line 85) | public static Long getExpectedRequestReceiveTimeEpoch() { return 1...
method getExpectedReferrer (line 86) | public static String getExpectedReferrer() { return "...
method getExpectedScreenResolution (line 87) | public static String getExpectedScreenResolution() { return "...
method getExpectedScreenWidth (line 88) | public static Long getExpectedScreenWidth() { return 1...
method getExpectedScreenHeight (line 89) | public static Long getExpectedScreenHeight() { return 8...
method getExpectedGoogleQuery (line 90) | public static String getExpectedGoogleQuery() { return "...
method getExpectedBui (line 91) | public static String getExpectedBui() { return "...
method getExpectedUseragent (line 92) | public static String getExpectedUseragent() { return "...
method getExpectedAsnNumber (line 94) | public static String getExpectedAsnNumber() { return "...
method getExpectedAsnOrganization (line 95) | public static String getExpectedAsnOrganization() { return "...
method getExpectedIspName (line 96) | public static String getExpectedIspName() { return "...
method getExpectedIspOrganization (line 97) | public static String getExpectedIspOrganization() { return "...
method getExpectedContinentName (line 99) | public static String getExpectedContinentName() { return "...
method getExpectedContinentCode (line 100) | public static String getExpectedContinentCode() { return "...
method getExpectedCountryName (line 101) | public static String getExpectedCountryName() { return "...
method getExpectedCountryIso (line 102) | public static String getExpectedCountryIso() { return "...
method getExpectedSubdivisionName (line 103) | public static String getExpectedSubdivisionName() { return "...
method getExpectedSubdivisionIso (line 104) | public static String getExpectedSubdivisionIso() { return "...
method getExpectedCityName (line 105) | public static String getExpectedCityName() { return "...
method getExpectedPostalCode (line 106) | public static String getExpectedPostalCode() { return "...
method getExpectedLocationLatitude (line 107) | public static Double getExpectedLocationLatitude() { return 5...
method getExpectedLocationLongitude (line 108) | public static Double getExpectedLocationLongitude() { return 5...
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/ExpectedClick.java
class ExpectedClick (line 42) | public class ExpectedClick {
method create (line 44) | public static Click create(){
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvro.java
class TestParserDoFnAvro (line 53) | @RunWith(JUnit4.class)
class ClickSetter (line 56) | public static class ClickSetter implements Builder<Click> {
method setRequestReceiveTime (line 60) | @Field("TIME.EPOCH:request.receive.time.epoch") public v...
method setScreenWidth (line 62) | @Field("SCREENWIDTH:request.firstline.uri.query.s.width") public v...
method setScreenHeight (line 63) | @Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public v...
method setUseragent (line 65) | @Field("HTTP.USERAGENT:request.user-agent") public v...
method setConnectionClientHost (line 67) | @Field("IP:connection.client.host") public v...
method setAsnNumber (line 69) | @Field("ASN:connection.client.host.asn.number") public v...
method setAsnOrganization (line 70) | @Field("STRING:connection.client.host.asn.organization") public v...
method setIspName (line 71) | @Field("STRING:connection.client.host.isp.name") public v...
method setIspOrganization (line 72) | @Field("STRING:connection.client.host.isp.organization") public v...
method setContinentName (line 74) | @Field("STRING:connection.client.host.continent.name") public v...
method setContinentCode (line 75) | @Field("STRING:connection.client.host.continent.code") public v...
method setCountryName (line 76) | @Field("STRING:connection.client.host.country.name") public v...
method setCountryIso (line 77) | @Field("STRING:connection.client.host.country.iso") public v...
method setSubdivisionName (line 78) | @Field("STRING:connection.client.host.subdivision.name") public v...
method setSubdivisionIso (line 79) | @Field("STRING:connection.client.host.subdivision.iso") public v...
method setCityName (line 80) | @Field("STRING:connection.client.host.city.name") public v...
method setPostalCode (line 81) | @Field("STRING:connection.client.host.postal.code") public v...
method setLocationLatitude (line 82) | @Field("STRING:connection.client.host.location.latitude") public v...
method setLocationLongitude (line 83) | @Field("STRING:connection.client.host.location.longitude") public v...
method build (line 85) | @Override
class MyParserDoFn (line 91) | public static class MyParserDoFn extends DoFn<String, Click> {
method setup (line 94) | @Setup
method processElement (line 105) | @ProcessElement
method testClassDefinitionAvro (line 122) | @Test
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvroInline.java
class TestParserDoFnAvroInline (line 53) | @RunWith(JUnit4.class)
class ClickSetter (line 56) | public static class ClickSetter implements Builder<Click> {
method setRequestReceiveTime (line 60) | @Field("TIME.EPOCH:request.receive.time.epoch") public v...
method setScreenWidth (line 62) | @Field("SCREENWIDTH:request.firstline.uri.query.s.width") public v...
method setScreenHeight (line 63) | @Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public v...
method setUseragent (line 65) | @Field("HTTP.USERAGENT:request.user-agent") public v...
method setConnectionClientHost (line 67) | @Field("IP:connection.client.host") public v...
method setAsnNumber (line 69) | @Field("ASN:connection.client.host.asn.number") public v...
method setAsnOrganization (line 70) | @Field("STRING:connection.client.host.asn.organization") public v...
method setIspName (line 71) | @Field("STRING:connection.client.host.isp.name") public v...
method setIspOrganization (line 72) | @Field("STRING:connection.client.host.isp.organization") public v...
method setContinentName (line 74) | @Field("STRING:connection.client.host.continent.name") public v...
method setContinentCode (line 75) | @Field("STRING:connection.client.host.continent.code") public v...
method setCountryName (line 76) | @Field("STRING:connection.client.host.country.name") public v...
method setCountryIso (line 77) | @Field("STRING:connection.client.host.country.iso") public v...
method setSubdivisionName (line 78) | @Field("STRING:connection.client.host.subdivision.name") public v...
method setSubdivisionIso (line 79) | @Field("STRING:connection.client.host.subdivision.iso") public v...
method setCityName (line 80) | @Field("STRING:connection.client.host.city.name") public v...
method setPostalCode (line 81) | @Field("STRING:connection.client.host.postal.code") public v...
method setLocationLatitude (line 82) | @Field("STRING:connection.client.host.location.latitude") public v...
method setLocationLongitude (line 83) | @Field("STRING:connection.client.host.location.longitude") public v...
method build (line 85) | @Override
method testClassDefinitionAvro (line 94) | @Test
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/MyRecord.java
class MyRecord (line 54) | @ToString
method assertIsValid (line 84) | public void assertIsValid() {
method setFullValid (line 112) | public MyRecord setFullValid() {
method checkTestMethodsPass (line 143) | @Test
method checkTestMethodsFail (line 149) | @Test(expected = AssertionError.class)
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/TestParserDoFnClass.java
class TestParserDoFnClass (line 42) | @RunWith(JUnit4.class)
class MyParserDoFn (line 45) | public static class MyParserDoFn extends DoFn<String, MyRecord> {
method MyParserDoFn (line 48) | public MyParserDoFn() throws NoSuchMethodException {
method processElement (line 53) | @ProcessElement
method testClassDefinition (line 62) | @Test
FILE: examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/TestParserDoFnInline.java
class TestParserDoFnInline (line 40) | public class TestParserDoFnInline implements Serializable {
method testInlineDefinition (line 45) | @Test
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/TestCase.java
class TestCase (line 31) | public final class TestCase {
method getLogFormat (line 37) | public static String getLogFormat() {
method getInputLine (line 41) | public static String getInputLine() {
method createTestParser (line 45) | public static Parser<MyRecord> createTestParser() throws NoSuchMethodE...
method getExpectedConnectionClientHost (line 85) | public static String getExpectedConnectionClientHost() { return "...
method getExpectedRequestReceiveTime (line 86) | public static String getExpectedRequestReceiveTime() { return "...
method getExpectedRequestReceiveTimeEpoch (line 87) | public static Long getExpectedRequestReceiveTimeEpoch() { return 1...
method getExpectedReferrer (line 88) | public static String getExpectedReferrer() { return "...
method getExpectedScreenResolution (line 89) | public static String getExpectedScreenResolution() { return "...
method getExpectedScreenWidth (line 90) | public static Long getExpectedScreenWidth() { return 1...
method getExpectedScreenHeight (line 91) | public static Long getExpectedScreenHeight() { return 8...
method getExpectedGoogleQuery (line 92) | public static String getExpectedGoogleQuery() { return "...
method getExpectedBui (line 93) | public static String getExpectedBui() { return "...
method getExpectedUseragent (line 94) | public static String getExpectedUseragent() { return "...
method getExpectedAsnNumber (line 96) | public static String getExpectedAsnNumber() { return "...
method getExpectedAsnOrganization (line 97) | public static String getExpectedAsnOrganization() { return "...
method getExpectedIspName (line 98) | public static String getExpectedIspName() { return "...
method getExpectedIspOrganization (line 99) | public static String getExpectedIspOrganization() { return "...
method getExpectedContinentName (line 101) | public static String getExpectedContinentName() { return "...
method getExpectedContinentCode (line 102) | public static String getExpectedContinentCode() { return "...
method getExpectedCountryName (line 103) | public static String getExpectedCountryName() { return "...
method getExpectedCountryIso (line 104) | public static String getExpectedCountryIso() { return "...
method getExpectedSubdivisionName (line 105) | public static String getExpectedSubdivisionName() { return "...
method getExpectedSubdivisionIso (line 106) | public static String getExpectedSubdivisionIso() { return "...
method getExpectedCityName (line 107) | public static String getExpectedCityName() { return "...
method getExpectedPostalCode (line 108) | public static String getExpectedPostalCode() { return "...
method getExpectedLocationLatitude (line 109) | public static Double getExpectedLocationLatitude() { return 5...
method getExpectedLocationLongitude (line 110) | public static Double getExpectedLocationLongitude() { return 5...
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/ExpectedClick.java
class ExpectedClick (line 42) | public class ExpectedClick {
method create (line 44) | public static Click create(){
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/TestParserMapFunctionAvroClass.java
class TestParserMapFunctionAvroClass (line 45) | class TestParserMapFunctionAvroClass implements Serializable {
class ClickSetter (line 47) | public static class ClickSetter implements Builder<Click> {
method setRequestReceiveTime (line 51) | @Field("TIME.EPOCH:request.receive.time.epoch") public v...
method setScreenWidth (line 53) | @Field("SCREENWIDTH:request.firstline.uri.query.s.width") public v...
method setScreenHeight (line 54) | @Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public v...
method setUseragent (line 56) | @Field("HTTP.USERAGENT:request.user-agent") public v...
method setConnectionClientHost (line 58) | @Field("IP:connection.client.host") public v...
method setAsnNumber (line 60) | @Field("ASN:connection.client.host.asn.number") public v...
method setAsnOrganization (line 61) | @Field("STRING:connection.client.host.asn.organization") public v...
method setIspName (line 62) | @Field("STRING:connection.client.host.isp.name") public v...
method setIspOrganization (line 63) | @Field("STRING:connection.client.host.isp.organization") public v...
method setContinentName (line 65) | @Field("STRING:connection.client.host.continent.name") public v...
method setContinentCode (line 66) | @Field("STRING:connection.client.host.continent.code") public v...
method setCountryName (line 67) | @Field("STRING:connection.client.host.country.name") public v...
method setCountryIso (line 68) | @Field("STRING:connection.client.host.country.iso") public v...
method setSubdivisionName (line 69) | @Field("STRING:connection.client.host.subdivision.name") public v...
method setSubdivisionIso (line 70) | @Field("STRING:connection.client.host.subdivision.iso") public v...
method setCityName (line 71) | @Field("STRING:connection.client.host.city.name") public v...
method setPostalCode (line 72) | @Field("STRING:connection.client.host.postal.code") public v...
method setLocationLatitude (line 73) | @Field("STRING:connection.client.host.location.latitude") public v...
method setLocationLongitude (line 74) | @Field("STRING:connection.client.host.location.longitude") public v...
method build (line 76) | @Override
class MyParserMapper (line 82) | public static class MyParserMapper extends RichMapFunction<String, Cli...
method open (line 85) | @Override
method map (line 96) | @Override
method testClassDefinitionAvro (line 107) | @Test
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/TestParserMapFunctionAvroInline.java
class TestParserMapFunctionAvroInline (line 45) | class TestParserMapFunctionAvroInline implements Serializable {
class ClickSetter (line 47) | public static class ClickSetter implements Builder<Click> {
method setRequestReceiveTime (line 51) | @Field("TIME.EPOCH:request.receive.time.epoch") public v...
method setScreenWidth (line 53) | @Field("SCREENWIDTH:request.firstline.uri.query.s.width") public v...
method setScreenHeight (line 54) | @Field("SCREENHEIGHT:request.firstline.uri.query.s.height") public v...
method setUseragent (line 56) | @Field("HTTP.USERAGENT:request.user-agent") public v...
method setConnectionClientHost (line 58) | @Field("IP:connection.client.host") public v...
method setAsnNumber (line 60) | @Field("ASN:connection.client.host.asn.number") public v...
method setAsnOrganization (line 61) | @Field("STRING:connection.client.host.asn.organization") public v...
method setIspName (line 62) | @Field("STRING:connection.client.host.isp.name") public v...
method setIspOrganization (line 63) | @Field("STRING:connection.client.host.isp.organization") public v...
method setContinentName (line 65) | @Field("STRING:connection.client.host.continent.name") public v...
method setContinentCode (line 66) | @Field("STRING:connection.client.host.continent.code") public v...
method setCountryName (line 67) | @Field("STRING:connection.client.host.country.name") public v...
method setCountryIso (line 68) | @Field("STRING:connection.client.host.country.iso") public v...
method setSubdivisionName (line 69) | @Field("STRING:connection.client.host.subdivision.name") public v...
method setSubdivisionIso (line 70) | @Field("STRING:connection.client.host.subdivision.iso") public v...
method setCityName (line 71) | @Field("STRING:connection.client.host.city.name") public v...
method setPostalCode (line 72) | @Field("STRING:connection.client.host.postal.code") public v...
method setLocationLatitude (line 73) | @Field("STRING:connection.client.host.location.latitude") public v...
method setLocationLongitude (line 74) | @Field("STRING:connection.client.host.location.longitude") public v...
method build (line 76) | @Override
method testInlineDefinitionAvro (line 82) | @Test
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/MyRecord.java
class MyRecord (line 55) | @ToString
method assertIsValid (line 85) | public void assertIsValid() {
method setFullValid (line 113) | public MyRecord setFullValid() {
method checkTestMethodsPass (line 144) | @Test
method checkTestMethodsFail (line 150) | @Test
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/TestParserMapFunctionClass.java
class TestParserMapFunctionClass (line 34) | class TestParserMapFunctionClass implements Serializable {
class MyParserMapper (line 36) | public static class MyParserMapper extends RichMapFunction<String, MyR...
method open (line 39) | @Override
method map (line 44) | @Override
method testClassDefinition (line 50) | @Test
FILE: examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/TestParserMapFunctionInline.java
class TestParserMapFunctionInline (line 34) | class TestParserMapFunctionInline implements Serializable {
method testInlineDefinition (line 36) | @Test
FILE: examples/apache-hadoop-mapreduce/src/main/java/nl/basjes/hadoop/io/input/Wordcount.java
class Wordcount (line 41) | public class Wordcount extends Configured implements Tool {
method Wordcount (line 46) | public Wordcount(String logFormat) {
class TokenizerMapper (line 52) | public static class TokenizerMapper extends
method map (line 58) | @Override
method run (line 70) | @Override
method main (line 113) | public static void main(String[] args) throws Exception {
FILE: examples/java-pojo/src/main/java/nl/basjes/parse/Main.java
class Main (line 29) | public final class Main {
method Main (line 30) | private Main(){}
method printAllPossibles (line 34) | private void printAllPossibles(String logformat) throws NoSuchMethodEx...
method run (line 56) | private void run() throws InvalidDissectorException, MissingDissectors...
method main (line 92) | public static void main(final String[] args) throws Exception {
FILE: examples/java-pojo/src/main/java/nl/basjes/parse/MyRecord.java
class MyRecord (line 25) | public class MyRecord {
method setQueryDeepMany (line 29) | @Field("STRING:request.firstline.uri.query.*")
method setQueryImg (line 34) | @Field("STRING:request.firstline.uri.query.img")
method setIP (line 39) | @Field("IP:connection.client.host")
method setValue (line 44) | @Field({
method toString (line 61) | public String toString() {
method clear (line 71) | public void clear() {
FILE: httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ApacheHttpdLogfileInputFormat.java
class ApacheHttpdLogfileInputFormat (line 39) | public class ApacheHttpdLogfileInputFormat extends
method listPossibleFields (line 49) | public List<String> listPossibleFields(String logformat) {
method listPossibleFields (line 53) | public static List<String> listPossibleFields(String logformat, Map<St...
method getLogFormat (line 61) | public String getLogFormat() {
method getRequestedFields (line 65) | public Set<String> getRequestedFields() {
method getTypeRemappings (line 69) | public Map<String, Set<String>> getTypeRemappings() {
method getAdditionalDissectors (line 73) | public List<Dissector> getAdditionalDissectors() {
method ApacheHttpdLogfileInputFormat (line 77) | public ApacheHttpdLogfileInputFormat() {
method ApacheHttpdLogfileInputFormat (line 81) | public ApacheHttpdLogfileInputFormat(
method createRecordReader (line 95) | public ApacheHttpdLogfileRecordReader createRecordReader() {
method getRecordReader (line 103) | public ApacheHttpdLogfileRecordReader getRecordReader() {
method createRecordReader (line 110) | @Override
method isSplitable (line 116) | @Override
method setTypeRemappings (line 123) | public void setTypeRemappings(Map<String, Set<String>> newTypeRemappin...
FILE: httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ApacheHttpdLogfileRecordReader.java
class ApacheHttpdLogfileRecordReader (line 46) | @SuppressWarnings({ "PMD.OnlyOneReturn", "PMD.BeanMembersShouldSerialize...
method ApacheHttpdLogfileRecordReader (line 70) | @SuppressWarnings("unused") // Used by the Hadoop framework
method ApacheHttpdLogfileRecordReader (line 75) | public ApacheHttpdLogfileRecordReader(String logformat,
method addRequestedFields (line 87) | private void addRequestedFields(Set<String> newRequestedFields) throws...
method setLogFormat (line 97) | private void setLogFormat(String newLogformat) {
method initialize (line 112) | @Override
method instantiateParser (line 154) | protected Parser<ParsedRecord> instantiateParser(String logFormat) {
method setupFields (line 161) | private void setupFields() throws MissingDissectorsException, InvalidD...
method getCasts (line 178) | public EnumSet<Casts> getCasts(String name) throws IOException {
method getParser (line 189) | public Parser<ParsedRecord> getParser() throws IOException {
method createParser (line 196) | private Parser<ParsedRecord> createParser() throws IOException {
method nextKeyValue (line 231) | @Override
method getCurrentKey (line 282) | @Override
method getCurrentValue (line 289) | @Override
method getProgress (line 294) | @Override
method close (line 301) | @Override
FILE: httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ParsedRecord.java
class ParsedRecord (line 27) | public class ParsedRecord implements Writable {
method equals (line 35) | @Override
method hashCode (line 55) | @Override
method write (line 64) | @Override
method readFields (line 101) | @Override
method ParsedRecord (line 142) | public ParsedRecord() {
method clear (line 145) | public void clear() {
method set (line 154) | public void set(String name, String value) {
method set (line 160) | public void set(String name, Long value) {
method set (line 166) | public void set(String name, Double value) {
method declareRequestedFieldname (line 178) | public void declareRequestedFieldname(String name) {
method setMultiValueString (line 185) | public void setMultiValueString(String name, String value) {
method getString (line 198) | public String getString(String name) {
method getLong (line 202) | public Long getLong(String name) {
method getDouble (line 206) | public Double getDouble(String name) {
method getStringSet (line 210) | public Map<String, String> getStringSet(String name) {
FILE: httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestApacheHttpdLogfileInputFormat.java
class TestApacheHttpdLogfileInputFormat (line 46) | class TestApacheHttpdLogfileInputFormat {
method checkInputFormat (line 51) | @Test
method checkAllOutputTypes (line 81) | @Test
FILE: httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestGetAllFields.java
class TestGetAllFields (line 32) | class TestGetAllFields {
method testGetAllField (line 34) | @Test
method checkPossibleFields (line 69) | @Test
FILE: httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestParsedRecord.java
class TestParsedRecord (line 33) | public class TestParsedRecord {
method serialize (line 36) | public static byte[] serialize(Writable writable) throws IOException {
method asWritable (line 48) | public static <T extends Writable> T asWritable(byte[] bytes, Class<T>...
method testParsedRecordSerialization (line 62) | @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "ObjectEqualsNul...
method setAllValues (line 85) | private void setAllValues(ParsedRecord record) {
method checkAllValues (line 114) | private void checkAllValues(ParsedRecord record) {
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdLogFormatDissector.java
class ApacheHttpdLogFormatDissector (line 47) | @SuppressWarnings({
method ApacheHttpdLogFormatDissector (line 57) | public ApacheHttpdLogFormatDissector(final String logFormat) {
method ApacheHttpdLogFormatDissector (line 62) | public ApacheHttpdLogFormatDissector() {
method overrideLogFormat (line 67) | private void overrideLogFormat(String originalLogformat, String logfor...
method setLogFormat (line 72) | @Override
method looksLikeApacheFormat (line 103) | public static boolean looksLikeApacheFormat(String logFormat) {
method makeHeaderNamesLowercaseInLogFormat (line 121) | protected String makeHeaderNamesLowercaseInLogFormat(String logformat) {
method removeModifiersFromLogformat (line 137) | protected String removeModifiersFromLogformat(String tokenLogFormat) {
method fixTimestampFormat (line 151) | protected String fixTimestampFormat(String tokenLogFormat) {
method cleanupLogFormat (line 161) | @Override
method decodeExtractedValue (line 169) | @Override
method createAllTokenParsers (line 201) | @Override
method addExtraOutput (line 642) | private void addExtraOutput(List<TokenParser> parsers,
method createFirstAndLastTokenParsers (line 653) | private List<TokenParser> createFirstAndLastTokenParsers(
method createFirstAndLastTokenParsers (line 662) | private List<TokenParser> createFirstAndLastTokenParsers(
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/HttpdLogFormatDissector.java
class HttpdLogFormatDissector (line 40) | public class HttpdLogFormatDissector extends Dissector {
method HttpdLogFormatDissector (line 51) | public HttpdLogFormatDissector() {
method HttpdLogFormatDissector (line 57) | public HttpdLogFormatDissector(final String multiLineLogFormat) {
method addAdditionalLogFormatsToHandleJettyUseragentProblem (line 73) | private void addAdditionalLogFormatsToHandleJettyUseragentProblem() {
method enableJettyFix (line 94) | public HttpdLogFormatDissector enableJettyFix() {
method addMultipleLogFormats (line 99) | public HttpdLogFormatDissector addMultipleLogFormats(final String mult...
method addLogFormat (line 103) | public HttpdLogFormatDissector addLogFormat(final List<String> logForm...
method addLogFormat (line 110) | public HttpdLogFormatDissector addLogFormat(final String logFormat) {
type LogFormatType (line 142) | private enum LogFormatType {
method determineMostLikelyLogFormat (line 148) | private LogFormatType determineMostLikelyLogFormat(final String logFor...
method initializeFromSettingsParameter (line 160) | @Override
method createAdditionalDissectors (line 166) | @Override
method dissect (line 173) | @Override
method getInputType (line 206) | @Override
method getPossibleOutput (line 211) | @Override
method prepareForDissect (line 225) | @Override
method prepareForRun (line 238) | @Override
method getAllLogFormats (line 254) | private List<String> getAllLogFormats() {
method initializeNewInstance (line 264) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/HttpdLoglineParser.java
class HttpdLoglineParser (line 34) | public class HttpdLoglineParser<RECORD> extends Parser<RECORD> {
method HttpdLoglineParser (line 40) | public HttpdLoglineParser(
method logVersion (line 50) | public static void logVersion(){
method padding (line 73) | private static String padding(char letter, int count) {
method logLine (line 81) | private static void logLine(String line, int width) {
method getVersion (line 87) | public static String getVersion() {
method HttpdLoglineParser (line 92) | public HttpdLoglineParser(
method setupDissectors (line 100) | private void setupDissectors(
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java
class NginxHttpdLogFormatDissector (line 49) | @SuppressWarnings({
method NginxHttpdLogFormatDissector (line 59) | public NginxHttpdLogFormatDissector(final String logFormat) {
method NginxHttpdLogFormatDissector (line 64) | public NginxHttpdLogFormatDissector() {
method overrideLogFormat (line 69) | private void overrideLogFormat(String originalLogformat, String logfor...
method setLogFormat (line 74) | @Override
method looksLikeNginxFormat (line 93) | public static boolean looksLikeNginxFormat(String logFormat) {
method decodeExtractedValue (line 107) | @Override
method createAllTokenParsers (line 132) | @Override
method createAdditionalDissectors (line 140) | @Override
class BinaryIPDissector (line 151) | public static class BinaryIPDissector extends SimpleDissector {
method BinaryIPDissector (line 157) | public BinaryIPDissector() {
method dissect (line 166) | @Override
class NotYetImplemented (line 181) | @Deprecated
method NotYetImplemented (line 184) | public NotYetImplemented(final String nLogFormatToken) {
method NotYetImplemented (line 188) | public NotYetImplemented(final String nLogFormatToken, final String ...
method NotYetImplemented (line 192) | public NotYetImplemented(final String nLogFormatToken, final String ...
method NotYetImplemented (line 196) | public NotYetImplemented(final String nLogFormatToken, final int pri...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/Utils.java
class Utils (line 32) | public final class Utils {
method Utils (line 34) | private Utils() {}
method resilientUrlDecode (line 46) | public static String resilientUrlDecode(String input) {
method hexCharsToByte (line 75) | public static byte hexCharsToByte(String twoHexDigits) {
method hexCharsToByte (line 82) | public static byte hexCharsToByte(char c1, char c2){
method decodeApacheHTTPDLogValue (line 154) | public static String decodeApacheHTTPDLogValue(String input){
method htmlEntityToURLEncoded (line 212) | private static String htmlEntityToURLEncoded(String entity) {
method makeHTMLEncodedInert (line 250) | public static String makeHTMLEncodedInert(String uriString) {
method replaceString (line 273) | public static String replaceString(
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpFirstLineDissector.java
class HttpFirstLineDissector (line 35) | public class HttpFirstLineDissector extends Dissector {
method getInputType (line 68) | @Override
method getPossibleOutput (line 75) | @Override
method dissect (line 86) | @Override
method outputDissection (line 124) | private void outputDissection(Parsable<?> parsable,
method prepareForDissect (line 140) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpFirstLineProtocolDissector.java
class HttpFirstLineProtocolDissector (line 33) | public class HttpFirstLineProtocolDissector extends Dissector {
method getInputType (line 38) | @Override
method getPossibleOutput (line 45) | @Override
method dissect (line 55) | @Override
method outputDissection (line 79) | private void outputDissection(Parsable<?> parsable,
method prepareForDissect (line 94) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpUriDissector.java
class HttpUriDissector (line 42) | public class HttpUriDissector extends Dissector {
method getInputType (line 47) | @Override
method getPossibleOutput (line 54) | @Override
method prepareForDissect (line 77) | @Override
method dissect (line 147) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ModUniqueIdDissector.java
class ModUniqueIdDissector (line 44) | public class ModUniqueIdDissector extends Dissector {
method getInputType (line 49) | @Override
method getPossibleOutput (line 56) | @Override
method prepareForDissect (line 76) | @Override
method dissect (line 104) | @Override
class UniqueIdRec (line 136) | private static final class UniqueIdRec {
method decodeToBytes (line 152) | private byte[] decodeToBytes(String modUniqueIdString) {
method decode (line 193) | private UniqueIdRec decode(String modUniqueIdString) {
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/QueryStringFieldDissector.java
class QueryStringFieldDissector (line 34) | public class QueryStringFieldDissector extends Dissector {
method getInputType (line 39) | @Override
method getPossibleOutput (line 47) | @Override
method prepareForDissect (line 58) | @Override
method prepareForRun (line 68) | @Override
method dissect (line 75) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/RequestCookieListDissector.java
class RequestCookieListDissector (line 35) | public class RequestCookieListDissector extends Dissector {
method getInputType (line 40) | @Override
method getPossibleOutput (line 48) | @Override
method prepareForDissect (line 59) | @Override
method prepareForRun (line 68) | @Override
method dissect (line 78) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ResponseSetCookieDissector.java
class ResponseSetCookieDissector (line 35) | public class ResponseSetCookieDissector extends Dissector {
method getInputType (line 40) | @Override
method getPossibleOutput (line 48) | @Override
method prepareForDissect (line 62) | @Override
method dissect (line 79) | @Override
method parseExpire (line 134) | private Long parseExpire(String expireString) {
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ResponseSetCookieListDissector.java
class ResponseSetCookieListDissector (line 34) | public class ResponseSetCookieListDissector extends Dissector {
method getInputType (line 39) | @Override
method getPossibleOutput (line 47) | @Override
method prepareForDissect (line 57) | @Override
method prepareForRun (line 67) | @Override
method dissect (line 79) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ScreenResolutionDissector.java
class ScreenResolutionDissector (line 32) | public class ScreenResolutionDissector extends Dissector {
method initializeFromSettingsParameter (line 39) | @Override
method dissect (line 47) | @Override
method getInputType (line 67) | @Override
method getPossibleOutput (line 72) | @Override
method prepareForDissect (line 80) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeStampDissector.java
class StrfTimeStampDissector (line 33) | public class StrfTimeStampDissector extends Dissector {
method StrfTimeStampDissector (line 40) | public StrfTimeStampDissector() {
method setDateTimePattern (line 44) | public void setDateTimePattern(String newDateTimePattern) {
method initializeFromSettingsParameter (line 58) | @Override
method dissect (line 64) | @Override
method getInputType (line 70) | @Override
method getPossibleOutput (line 75) | @Override
method prepareForDissect (line 80) | @Override
method prepareForRun (line 85) | @Override
method initializeNewInstance (line 90) | @Override
method setInputType (line 98) | @Override
method createAdditionalDissectors (line 103) | @Override
class LocalizedTimeDissector (line 108) | public static class LocalizedTimeDissector extends Dissector {
method LocalizedTimeDissector (line 112) | public LocalizedTimeDissector() {
method LocalizedTimeDissector (line 115) | public LocalizedTimeDissector(String inputType) {
method setInputType (line 119) | @Override
method initializeFromSettingsParameter (line 124) | @Override
method dissect (line 130) | @Override
method getInputType (line 136) | @Override
method getPossibleOutput (line 141) | @Override
method prepareForDissect (line 148) | @Override
method initializeNewInstance (line 153) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeToDateTimeFormatter.java
class StrfTimeToDateTimeFormatter (line 47) | public final class StrfTimeToDateTimeFormatter extends StrfTimeBaseListe...
method convert (line 53) | public static DateTimeFormatter convert(String strfformat) {
method convert (line 57) | public static DateTimeFormatter convert(String strfformat, ZoneId defa...
method StrfTimeToDateTimeFormatter (line 90) | private StrfTimeToDateTimeFormatter(String inputStrfformat, ZoneId new...
method build (line 97) | public DateTimeFormatter build() {
method hasSyntaxError (line 110) | public boolean hasSyntaxError() {
method syntaxError (line 114) | @Override
method reportAmbiguity (line 119) | @Override
method reportAttemptingFullContext (line 124) | @Override
method reportContextSensitivity (line 129) | @Override
class UnsupportedStrfField (line 134) | public static class UnsupportedStrfField extends RuntimeException {
method UnsupportedStrfField (line 135) | public UnsupportedStrfField(String s) {
method enterMsecFrac (line 142) | @Override
method enterUsecFrac (line 148) | @Override
method enterText (line 154) | @Override
method enterTab (line 159) | @Override
method enterPercent (line 164) | @Override
method enterNewline (line 169) | @Override
method enterPa (line 174) | @Override
method enterPA (line 180) | @Override
method enterPb (line 186) | @Override
method enterPB (line 193) | @Override
method enterPc (line 199) | @Override
method enterPC (line 205) | @Override
method enterPd (line 210) | @Override
method enterPD (line 216) | @Override
method enterPe (line 227) | @Override
method enterPF (line 233) | @Override
method enterPG (line 244) | @Override
method enterPg (line 253) | @Override
method enterPH (line 259) | @Override
method enterPI (line 265) | @Override
method enterPj (line 271) | @Override
method enterPk (line 277) | @Override
method enterPl (line 284) | @Override
method enterPm (line 291) | @Override
method enterPM (line 297) | @Override
method enterPp (line 303) | @Override
method enterPP (line 316) | @Override
method enterPr (line 322) | @Override
method enterPR (line 335) | @Override
method enterPs (line 344) | @Override
method enterPS (line 352) | @Override
method enterPT (line 359) | @Override
method enterPu (line 370) | @Override
method enterPU (line 376) | @Override
method enterPV (line 383) | @Override
method enterPw (line 390) | @Override
method enterPW (line 396) | @Override
method enterPx (line 403) | @Override
method enterPX (line 409) | @Override
method enterPy (line 415) | @Override
method enterPY (line 421) | @Override
method enterPz (line 427) | @Override
method enterPZ (line 434) | @Override
method enterPplus (line 441) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/TimeStampDissector.java
class TimeStampDissector (line 44) | public class TimeStampDissector extends Dissector {
method TimeStampDissector (line 59) | @SuppressWarnings("UnusedDeclaration")
method TimeStampDissector (line 64) | public TimeStampDissector(String newDateTimePattern) {
method TimeStampDissector (line 68) | public TimeStampDissector(String inputType, String newDateTimePattern) {
method setLocale (line 79) | public TimeStampDissector setLocale(Locale newLocale) {
method getLocale (line 85) | public Locale getLocale() {
method initializeFromSettingsParameter (line 90) | @Override
method setDateTimePattern (line 99) | public void setDateTimePattern(String nDateTimePattern) {
method setFormatter (line 103) | protected void setFormatter(DateTimeFormatter newFormatter) {
method getFormatter (line 107) | protected DateTimeFormatter getFormatter() {
method initializeNewInstance (line 118) | @Override
method getInputType (line 130) | @Override
method setInputType (line 135) | @Override
method getPossibleOutput (line 142) | @Override
method prepareForDissect (line 229) | @Override
method prepareForRun (line 363) | @Override
method dissect (line 410) | @Override
method parse (line 416) | private ZonedDateTime parse(String fieldValue) throws DateTimeParseExc...
method dissect (line 429) | protected void dissect(ParsedField field, final Parsable<?> parsable, ...
method attemptRecoverParseError (line 654) | private String attemptRecoverParseError(String fieldValue, DateTimePar...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/AbstractGeoIPDissector.java
class AbstractGeoIPDissector (line 35) | public abstract class AbstractGeoIPDissector extends Dissector {
method AbstractGeoIPDissector (line 41) | public AbstractGeoIPDissector() {
method AbstractGeoIPDissector (line 44) | public AbstractGeoIPDissector(String databaseFileName) {
method getInputType (line 48) | @Override
method initializeFromSettingsParameter (line 55) | @Override
method initializeNewInstance (line 63) | @Override
method prepareForRun (line 72) | @Override
method openDatabaseFile (line 86) | protected InputStream openDatabaseFile(String filename) throws FileNot...
method dissect (line 93) | @Override
method dissect (line 114) | abstract void dissect(Parsable<?> parsable, String inputname, InetAddr...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPASNDissector.java
class GeoIPASNDissector (line 36) | public class GeoIPASNDissector extends AbstractGeoIPDissector {
method GeoIPASNDissector (line 38) | @SuppressWarnings("unused") // Used via reflection
method GeoIPASNDissector (line 43) | public GeoIPASNDissector(String databaseFileName) {
method getPossibleOutput (line 47) | @Override
method prepareForDissect (line 60) | @Override
method dissect (line 80) | public void dissect(final Parsable<?> parsable, final String inputname...
method extractAsnFields (line 91) | protected void extractAsnFields(final Parsable<?> parsable, final Stri...
method extractAsnFields (line 101) | protected void extractAsnFields(final Parsable<?> parsable, final Stri...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPCityDissector.java
class GeoIPCityDissector (line 39) | public class GeoIPCityDissector extends GeoIPCountryDissector {
method GeoIPCityDissector (line 41) | @SuppressWarnings("unused") // Used via reflection
method GeoIPCityDissector (line 46) | public GeoIPCityDissector(String databaseFileName) {
method getPossibleOutput (line 50) | @Override
method prepareForDissect (line 92) | @Override
method dissect (line 171) | public void dissect(final Parsable<?> parsable, final String inputname...
method extractCityResponseFields (line 183) | protected void extractCityResponseFields(final Parsable<?> parsable, f...
method extractSubdivisionFields (line 200) | protected void extractSubdivisionFields(final Parsable<?> parsable, fi...
method extractCityFields (line 211) | protected void extractCityFields(final Parsable<?> parsable, final Str...
method extractPostalFields (line 225) | protected void extractPostalFields(final Parsable<?> parsable, final S...
method extractLocationFields (line 236) | protected void extractLocationFields(final Parsable<?> parsable, final...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPCountryDissector.java
class GeoIPCountryDissector (line 38) | public class GeoIPCountryDissector extends AbstractGeoIPDissector {
method GeoIPCountryDissector (line 40) | @SuppressWarnings("unused") // Used via reflection
method GeoIPCountryDissector (line 45) | public GeoIPCountryDissector(String databaseFileName) {
method getPossibleOutput (line 49) | @Override
method prepareForDissect (line 73) | @Override
method dissect (line 115) | public void dissect(final Parsable<?> parsable, final String inputname...
method extractCountryResponseFields (line 126) | protected void extractCountryResponseFields(final Parsable<?> parsable...
method extractCityResponseFields (line 138) | protected void extractCityResponseFields(final Parsable<?> parsable, f...
method extractContinentFields (line 147) | protected void extractContinentFields(final Parsable<?> parsable, fina...
method extractCountryFields (line 159) | protected void extractCountryFields(final Parsable<?> parsable, final ...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPISPDissector.java
class GeoIPISPDissector (line 33) | public class GeoIPISPDissector extends GeoIPASNDissector {
method GeoIPISPDissector (line 35) | @SuppressWarnings("unused") // Used via reflection
method GeoIPISPDissector (line 40) | public GeoIPISPDissector(String databaseFileName) {
method getPossibleOutput (line 44) | @Override
method prepareForDissect (line 57) | @Override
method dissect (line 81) | public void dissect(final Parsable<?> parsable, final String inputname...
method extractIspFields (line 93) | protected void extractIspFields(final Parsable<?> parsable, final Stri...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/CoreLogModule.java
class CoreLogModule (line 43) | public class CoreLogModule implements NginxModule {
method getTokenParsers (line 44) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/GeoIPModule.java
class GeoIPModule (line 31) | public class GeoIPModule implements NginxModule {
method getTokenParsers (line 35) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/KubernetesIngressModule.java
class KubernetesIngressModule (line 31) | public class KubernetesIngressModule implements NginxModule {
method getTokenParsers (line 35) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/NginxModule.java
type NginxModule (line 26) | public interface NginxModule {
method getTokenParsers (line 27) | List<TokenParser> getTokenParsers();
method getDissectors (line 29) | default List<Dissector> getDissectors() {
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/SslModule.java
class SslModule (line 33) | public class SslModule implements NginxModule {
method getTokenParsers (line 37) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/UpstreamListDissector.java
class UpstreamListDissector (line 49) | public class UpstreamListDissector extends Dissector {
method UpstreamListDissector (line 60) | public UpstreamListDissector() {
method UpstreamListDissector (line 68) | public UpstreamListDissector(String inputType,
method dissect (line 78) | @Override
method getInputType (line 115) | @Override
method getPossibleOutput (line 120) | @Override
method prepareForDissect (line 131) | @Override
method initializeNewInstance (line 144) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/UpstreamModule.java
class UpstreamModule (line 38) | public class UpstreamModule implements NginxModule {
method upstreamListOf (line 42) | private String upstreamListOf(String regex) {
method optionalUpstreamListOf (line 46) | private String optionalUpstreamListOf(String regex) {
method getTokenParsers (line 50) | @Override
method getDissectors (line 181) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/VariousModule.java
class VariousModule (line 33) | public class VariousModule implements NginxModule {
method getTokenParsers (line 37) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/NamedTokenParser.java
class NamedTokenParser (line 28) | public class NamedTokenParser extends TokenParser {
method NamedTokenParser (line 34) | public NamedTokenParser(
method NamedTokenParser (line 43) | public NamedTokenParser(
method getNextToken (line 58) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/ParameterizedTokenParser.java
class ParameterizedTokenParser (line 35) | public class ParameterizedTokenParser extends TokenParser {
method ParameterizedTokenParser (line 41) | public ParameterizedTokenParser(
method addOutputField (line 55) | @Override
method addOutputFields (line 63) | @Override
method getNextToken (line 73) | @Override
method tokenParameterToTypeName (line 114) | String tokenParameterToTypeName(String parameter) {
method stringHashAsHexString (line 122) | private String stringHashAsHexString(String input) {
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/Token.java
class Token (line 30) | public class Token implements Serializable {
method Token (line 43) | public Token(
method addOutputField (line 54) | public Token addOutputField(String type, String name, EnumSet<Casts> c...
method addOutputFields (line 59) | public Token addOutputFields(List<TokenOutputField> nOutputFields) {
method getOutputFields (line 64) | public List<TokenOutputField> getOutputFields() {
method canProduceADesiredFieldName (line 68) | public boolean canProduceADesiredFieldName(Set<String> desiredNames) {
method setCustomDissector (line 77) | public void setCustomDissector(Dissector dissector) {
method getCustomDissector (line 81) | public Dissector getCustomDissector() {
method getRegex (line 85) | public String getRegex() {
method getStartPos (line 89) | public int getStartPos() {
method getLength (line 93) | public int getLength() {
method getPrio (line 98) | public int getPrio() {
method setWarningMessageWhenUsed (line 102) | public void setWarningMessageWhenUsed(String message) {
method tokenWasUsed (line 106) | public void tokenWasUsed() {
method toString (line 115) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenFormatDissector.java
class TokenFormatDissector (line 39) | @SuppressWarnings({
class FixedStringTokenParser (line 60) | public static class FixedStringTokenParser extends TokenParser {
method FixedStringTokenParser (line 61) | public FixedStringTokenParser(final String nLogFormatToken, final St...
method getNextToken (line 65) | @Override
class FixedStringToken (line 81) | public static class FixedStringToken extends Token {
method FixedStringToken (line 82) | public FixedStringToken(String nRegex, int nStartPos, int nLength, i...
class NotImplementedTokenParser (line 89) | public static class NotImplementedTokenParser extends TokenParser {
method NotImplementedTokenParser (line 91) | public NotImplementedTokenParser(final String nLogFormatToken, final...
method NotImplementedTokenParser (line 95) | public NotImplementedTokenParser(final String nLogFormatToken, final...
method TokenFormatDissector (line 105) | public TokenFormatDissector(final String logFormat) {
method TokenFormatDissector (line 109) | public TokenFormatDissector() {
method initializeFromSettingsParameter (line 112) | @Override
method initializeNewInstance (line 118) | @Override
method setLogFormat (line 127) | public void setLogFormat(final String logformat) {
method getLogFormat (line 149) | public String getLogFormat() {
method getLogFormatRegEx (line 153) | @SuppressWarnings("unused") // Useful for debugging purposes
method prepareForDissect (line 162) | @Override
method prepareForRun (line 178) | @Override
method setInputType (line 217) | @Override
method getInputType (line 224) | @Override
method getPossibleOutput (line 229) | @Override
method decodeExtractedValue (line 240) | public abstract String decodeExtractedValue(String tokenName, String v...
method dissect (line 242) | @Override
method cleanupLogFormat (line 285) | protected String cleanupLogFormat(String tokenLogFormat){
method parseTokenLogFileDefinition (line 290) | @SuppressWarnings({ "PMD.AvoidInstantiatingObjectsInLoops",
method createAdditionalDissectors (line 382) | @Override
method createAllTokenParsers (line 390) | protected abstract List<TokenParser> createAllTokenParsers();
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenOutputField.java
class TokenOutputField (line 26) | public class TokenOutputField implements Serializable {
method TokenOutputField (line 39) | public TokenOutputField(String type, String name, EnumSet<Casts> casts) {
method getType (line 46) | public String getType() {
method getName (line 50) | public String getName() {
method getCasts (line 54) | public EnumSet<Casts> getCasts() {
method deprecateFor (line 58) | public TokenOutputField deprecateFor(String deprecatedFor) {
method isDeprecated (line 63) | public boolean isDeprecated() {
method wasUsed (line 67) | public void wasUsed() {
method toString (line 75) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenParser.java
class TokenParser (line 30) | public class TokenParser implements Serializable {
method TokenParser (line 77) | public TokenParser(final String nLogFormatToken,
method TokenParser (line 85) | public TokenParser(final String nLogFormatToken,
method TokenParser (line 94) | public TokenParser(final String nLogFormatToken,
method TokenParser (line 108) | public TokenParser(final String nLogFormatToken,
method TokenParser (line 113) | public TokenParser(final String nLogFormatToken,
method TokenParser (line 119) | public TokenParser(final String nLogFormatToken,
method addOutputField (line 129) | public TokenParser addOutputField(String type, String name, EnumSet<Ca...
method addOutputField (line 134) | public TokenParser addOutputField(String type, String name, EnumSet<Ca...
method addOutputField (line 139) | public TokenParser addOutputField(TokenOutputField outputField) {
method addOutputFields (line 144) | public TokenParser addOutputFields(List<TokenOutputField> nOutputField...
method getOutputFields (line 149) | public List<TokenOutputField> getOutputFields() {
method setWarningMessageWhenUsed (line 155) | public TokenParser setWarningMessageWhenUsed(String message) {
method getLogFormatToken (line 160) | public String getLogFormatToken() {
method getRegex (line 164) | public String getRegex() {
method getPrio (line 168) | public int getPrio() {
method getCustomDissector (line 172) | public Dissector getCustomDissector() {
method getNextToken (line 178) | public Token getNextToken(final String logFormat, final int startOffse...
method getTokens (line 206) | public List<Token> getTokens(final String logFormat) {
method addCustomDissector (line 227) | protected boolean addCustomDissector(Token token, String fieldType, St...
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenSorterByStartPos.java
class TokenSorterByStartPos (line 22) | public class TokenSorterByStartPos implements Comparator<Token>, Seriali...
method compare (line 23) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertCLFIntoNumber.java
class ConvertCLFIntoNumber (line 23) | public class ConvertCLFIntoNumber extends TypeConvertBaseDissector {
method ConvertCLFIntoNumber (line 24) | public ConvertCLFIntoNumber() {
method ConvertCLFIntoNumber (line 28) | public ConvertCLFIntoNumber(String nInputType, String nOutputType) {
method dissect (line 32) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertMillisecondsIntoMicroseconds.java
class ConvertMillisecondsIntoMicroseconds (line 23) | public class ConvertMillisecondsIntoMicroseconds extends TypeConvertBase...
method ConvertMillisecondsIntoMicroseconds (line 24) | public ConvertMillisecondsIntoMicroseconds() {
method ConvertMillisecondsIntoMicroseconds (line 28) | public ConvertMillisecondsIntoMicroseconds(String inputType, String nO...
method dissect (line 32) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertNumberIntoCLF.java
class ConvertNumberIntoCLF (line 23) | public class ConvertNumberIntoCLF extends TypeConvertBaseDissector {
method ConvertNumberIntoCLF (line 24) | public ConvertNumberIntoCLF() {
method ConvertNumberIntoCLF (line 28) | public ConvertNumberIntoCLF(String inputType, String nOutputType) {
method dissect (line 32) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertSecondsWithMillisStringDissector.java
class ConvertSecondsWithMillisStringDissector (line 23) | public class ConvertSecondsWithMillisStringDissector extends TypeConvert...
method ConvertSecondsWithMillisStringDissector (line 24) | public ConvertSecondsWithMillisStringDissector() {
method ConvertSecondsWithMillisStringDissector (line 28) | public ConvertSecondsWithMillisStringDissector(String inputType, Strin...
method dissect (line 32) | @Override
FILE: httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/TypeConvertBaseDissector.java
class TypeConvertBaseDissector (line 29) | public abstract class TypeConvertBaseDissector extends SimpleDissector {
method TypeConvertBaseDissector (line 32) | public TypeConvertBaseDissector() {
method fillOutputConfig (line 36) | private static HashMap<String, EnumSet<Casts>> fillOutputConfig(String...
method TypeConvertBaseDissector (line 42) | public TypeConvertBaseDissector(String nInputType, String nOutputType) {
method initializeNewInstance (line 48) | @Override
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ApacheHttpdAllFieldsTest.java
class ApacheHttpdAllFieldsTest (line 27) | class ApacheHttpdAllFieldsTest {
method verifyFieldAvailability (line 29) | private void verifyFieldAvailability(String logformat, String... expec...
method checkDeprecationMessage (line 41) | @Test
method testAllFieldsAvailability (line 60) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ApacheHttpdLogParserTest.java
class ApacheHttpdLogParserTest (line 38) | class ApacheHttpdLogParserTest {
class TestRecord (line 42) | public static class TestRecord {
method setValue (line 45) | @SuppressWarnings("UnusedDeclaration")
method getResults (line 80) | public Map<String, String> getResults() {
method fullTest1 (line 105) | @Test
method fullTest2 (line 169) | @Test
method fullTestTooLongUri (line 206) | @Test
class TestRecordMissing (line 245) | public static class TestRecordMissing {
method dummy (line 246) | @SuppressWarnings({"UnusedDeclaration", "EmptyMethod"})
method testMissing (line 252) | @Test
class TestRecordMissing2 (line 265) | public static class TestRecordMissing2 {
method dummy (line 266) | @SuppressWarnings({"UnusedDeclaration", "EmptyMethod"})
method testMissing2 (line 272) | @Test
method testGetPossiblePaths (line 285) | @Test
method testGetPossiblePathsWithUnusableLogFormat (line 300) | @Test
method testLogFormatCleanup (line 309) | @Test
method verifyCommonFormatNamesMapping (line 319) | @Test
class EmptyTestRecord (line 327) | public static class EmptyTestRecord extends HashMap<String, String> {
method put (line 328) | @Override
method testQueryStringDissector (line 335) | @Test
method test408ModReqTimeout (line 440) | @Test
method testFailOnMissingDissectors (line 474) | @Test
method testIgnoreMissingDissectors (line 491) | @Test
method testExternalExample (line 503) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/BasicOverallTest.java
class BasicOverallTest (line 29) | class BasicOverallTest {
class MyRecord (line 30) | public static class MyRecord {
method setValue (line 34) | @SuppressWarnings({"unused"}) // Used via reflection
method toString (line 39) | public String toString() {
method clear (line 50) | public void clear() {
method testBasicParsing (line 115) | @Test
method ensureAllOutputsAreThere (line 131) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ClientHintsTest.java
class ClientHintsTest (line 25) | class ClientHintsTest {
method testClientHintsWithEscapedDoubleQuotes (line 26) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/CookiesTest.java
class CookiesTest (line 31) | class CookiesTest {
class EmptyTestRecord (line 33) | private static final class EmptyTestRecord {
class TestRecord (line 36) | public static class TestRecord {
method setValue (line 41) | @SuppressWarnings({"unused"}) // Used via reflection
method getResults (line 86) | public Map<String, String> getResults() {
method setValueLong (line 90) | @SuppressWarnings({"unused"}) // Used via reflection
method getLongResults (line 105) | public Map<String, Long> getLongResults() {
method testEmptyRecordPossibles (line 131) | @Test
method testRecordPossibles (line 143) | @Test
method check (line 155) | private void check(String expect, Map<String, String> results, String ...
method cookiesTest (line 160) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/EdgeCasesTest.java
class EdgeCasesTest (line 26) | class EdgeCasesTest {
method testInvalidFirstLine (line 27) | @Test
method checkBadUri (line 61) | void checkBadUri(String logLine, String expectedUri) {
method testBadUri (line 79) | @Test
method checkErrorLogging (line 94) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/JettyLogFormatParserTest.java
class JettyLogFormatParserTest (line 29) | class JettyLogFormatParserTest {
class TestRecord (line 33) | public static class TestRecord {
method setValue (line 36) | @SuppressWarnings("UnusedDeclaration")
method getResults (line 54) | public Map<String, String> getResults() {
method buggyJettyLogline (line 61) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/JsonLogFormatTest.java
class JsonLogFormatTest (line 27) | class JsonLogFormatTest {
method testBasicParsing (line 46) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/MultiLineHttpdLogParserTest.java
class MultiLineHttpdLogParserTest (line 32) | class MultiLineHttpdLogParserTest {
class TestRecord (line 36) | public static class TestRecord {
method setValue (line 39) | @SuppressWarnings("UnusedDeclaration")
method getResults (line 53) | public Map<String, String> getResults() {
method fullTest1 (line 63) | @Test
method validateLine1 (line 92) | private void validateLine1(Parser<TestRecord> parser) throws InvalidDi...
method validateLine2 (line 111) | private void validateLine2(Parser<TestRecord> parser) throws InvalidDi...
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatJsonTest.java
class NginxLogFormatJsonTest (line 24) | class NginxLogFormatJsonTest {
method testJsonFormat (line 26) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java
class NginxLogFormatTest (line 37) | public class NginxLogFormatTest {
method testBasicLogFormat (line 41) | @Test
method testBasicLogFormatDissector (line 56) | @Test
method testBasicLogFormatWithUnknownField (line 77) | @Test
method testCompareApacheAndNginxOutput (line 95) | @Test
method testFullTestAllFields (line 207) | @Test
class SingleFieldTestcase (line 331) | private static class SingleFieldTestcase {
method SingleFieldTestcase (line 337) | SingleFieldTestcase(String logformat, String logline, String fieldNa...
method validateAllFields (line 345) | @Test
method validateAllFieldsPrefix (line 447) | @Test
method bugReport60 (line 471) | @Test
method bugReport227_bad (line 558) | @Test
method bugReport227_empty (line 590) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/UtilsTest.java
class UtilsTest (line 28) | class UtilsTest {
method testUrlDecoder (line 30) | @Test
method testHtmlEncoding (line 54) | @Test
method testHtmlEncodingFull (line 64) | @Test
method testHexToByte (line 128) | @Test
method testHexToByteIllegalLeft (line 155) | @Test
method testHexToByteIllegalRight (line 162) | @Test
method testApacheLogDecoder (line 169) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestCookieDissector.java
class TestCookieDissector (line 23) | class TestCookieDissector {
method testRequestCookies (line 25) | @Test
method testResponseSetCookies (line 43) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestGeoIPDissectors.java
class TestGeoIPDissectors (line 33) | class TestGeoIPDissectors {
method createTester (line 41) | DissectorTester createTester(Dissector dissector) {
class TestGeoIPDissectorsWithPrefix (line 47) | public static class TestGeoIPDissectorsWithPrefix extends TestGeoIPDis...
method createTester (line 49) | DissectorTester createTester(Dissector dissector) {
method testBadFileASN (line 59) | @Test
method testBadFileISP (line 69) | @Test
method testBadFileCity (line 79) | @Test
method testBadFileCountry (line 89) | @Test
method testUnknownIPASN (line 101) | @Test
method testUnknownIPISP (line 109) | @Test
method testUnknownIPCity (line 117) | @Test
method testUnknownIPCountry (line 125) | @Test
method testGeoIPASN (line 136) | @Test
method testGeoIPISP (line 146) | @Test
method testGeoIPCountry (line 158) | @Test
method testGeoIPCity (line 173) | @Test
method testGeoIPASNIpv6 (line 209) | @Test
method testGeoIPISPIpv6 (line 219) | @Test
method testGeoIPCountryIpv6 (line 231) | @Test
method testGeoIPCityIpv6 (line 246) | @Test
method testGeoIPISPLocalhost (line 282) | @Test
method testGeoIPASNLocalhost (line 294) | @Test
method testGeoIPCountryLocalhost (line 304) | @Test
method testGeoIPCityLocalhost (line 319) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestHttpFirstLineDissector.java
class TestHttpFirstLineDissector (line 23) | class TestHttpFirstLineDissector {
method testNormal (line 24) | @Test
method testChoppedFirstLine (line 37) | @Test
method testInvalidFirstLine (line 50) | @Test
method testStrangeCommandVersionControl (line 60) | @Test
method testProtocol (line 73) | @Test
method testChoppedProtocol (line 83) | @Test
method testEmptyProtocol1 (line 93) | @Test
method testEmptyProtocol2 (line 103) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestHttpUriDissector.java
class TestHttpUriDissector (line 23) | class TestHttpUriDissector {
method testFullUrl1 (line 25) | @Test
method testFullUrl2 (line 42) | @Test
method testFullUrl3 (line 59) | @Test
method testFullUrl4 (line 76) | @Test
method testFullUrl5 (line 93) | @Test
method testAndroidApp1 (line 110) | @Test
method testAndroidApp2 (line 127) | @Test
method testBadURI (line 144) | @Test
method testBadURIEncoding (line 162) | @Test
method testBadURIMultiPercentEncoding (line 181) | @Test
method testDoubleHashes (line 200) | @Test
method testHTMLEntities (line 215) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestModUniqueIdDissector.java
class TestModUniqueIdDissector (line 23) | class TestModUniqueIdDissector {
method testUniqueId1 (line 25) | @Test
method testUniqueId2 (line 46) | @Test
method testBadUniqueIdTooShort (line 67) | @Test
method testBadUniqueIdNotBase64 (line 80) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestQueryStringDissector.java
class TestQueryStringDissector (line 23) | class TestQueryStringDissector {
method testQueryString (line 25) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestTimeStampDissector.java
class TestTimeStampDissector (line 47) | class TestTimeStampDissector {
method ensureDefaultLocaleFollowsISOWeekFields (line 49) | @Test
method testTimeStampDissector (line 57) | @Test
method testTimeStampDissectorPossibles (line 100) | @Test
method testStrftimeStampDissectorPossibles (line 129) | @Test
method testTimeStamUpperLowerCaseVariations (line 164) | @Test
method locales (line 185) | private static Stream<Arguments> locales() {
method testTimeStampMonthNameVariations (line 195) | @ParameterizedTest(name = "Test {index}: {0} ({1})")
method testHandlingOfNotYetImplementedSpecialTimeFormat (line 249) | @Test
method testSpecialTimeFormat (line 263) | @Test
method testSpecialTimeFormatBegin (line 309) | @Test
method testSpecialTimeFormatEnd (line 318) | @Test
method testSpecialTimeFormatMultiFields1 (line 327) | @Test
method testSpecialTimeFormatMultiFields2 (line 367) | @Test
method testSpecialTimeLeadingSpaces1 (line 407) | @Test
method testSpecialTimeLeadingSpaces2a (line 419) | @Test
method testMultipleSpecialTime (line 431) | @Test
method testReportedSpecialTime (line 456) | @Test
method testAllStrfFieldsLowValues (line 468) | @Test
method testAllStrfFieldsHighValues (line 517) | @Test
method checkStrfField (line 567) | private void checkStrfField(ZonedDateTime dateTime, String strffield, ...
method ensureUnsupportedFields (line 578) | @Test
method checkUnsupported (line 589) | private void checkUnsupported(String strffield) {
method strfTimeWithMissingTimeZone (line 600) | @Test
method testStrfTimeMsecfrac (line 617) | @Test
method testStrfTimeMsecfracP (line 635) | @Test
method testStrfTimeUsecfrac (line 653) | @Test
method testStrfTimeUsecfracP (line 671) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/nginxmodules/NginxAllFieldsTest.java
class NginxAllFieldsTest (line 30) | class NginxAllFieldsTest {
method ensureAllFieldsAreHandled (line 33) | @Test
method checkVariable (line 291) | public void checkVariable(String variableName) {
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/nginxmodules/NginxUpstreamTest.java
class NginxUpstreamTest (line 31) | class NginxUpstreamTest {
class SingleFieldTestcase (line 35) | private static class SingleFieldTestcase {
method SingleFieldTestcase (line 41) | SingleFieldTestcase(String logformat, String logline, String fieldNa...
method testBasicLogFormat (line 49) | @Test
method testFullLine (line 92) | @Test
method validateAllFields (line 114) | @Test
FILE: httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/translate/TestTranslators.java
class TestTranslators (line 25) | class TestTranslators {
method testCLFToNumberMin (line 27) | @Test
method testCLFToNumber0 (line 36) | @Test
method testCLFToNumber1 (line 45) | @Test
method testNumberToCLF0 (line 54) | @Test
method testNumberToCLF1 (line 64) | @Test
FILE: httpdlog/httpdlog-serde/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdlogDeserializer.java
class ApacheHttpdlogDeserializer (line 104) | public class ApacheHttpdlogDeserializer extends AbstractSerDe {
class ColumnToGetterMapping (line 128) | static class ColumnToGetterMapping {
method initialize (line 136) | @Override
method getObjectInspector (line 267) | @Override
method deserialize (line 272) | @Override
method getSerializedClass (line 317) | @Override
method getSerDeStats (line 322) | @Override
FILE: httpdlog/httpdlog-serde/src/test/java/nl/basjes/parse/httpdlog/TestAllDissectorTypes.java
class TestAllDissectorTypes (line 35) | public class TestAllDissectorTypes {
method testAllDissectorOutputTypes (line 39) | @Test
method getTestSerDe (line 83) | private AbstractSerDe getTestSerDe() throws SerDeException {
FILE: httpdlog/httpdlog-serde/src/test/java/nl/basjes/parse/httpdlog/TestApacheHttpdlogDeserializer.java
class TestApacheHttpdlogDeserializer (line 38) | class TestApacheHttpdlogDeserializer {
method testBasicParse (line 59) | @Test
method testHighFailRatio1 (line 82) | @Test
method getTestSerDe (line 114) | private AbstractSerDe getTestSerDe() throws SerDeException {
FILE: parser-core/src/main/java/nl/basjes/parse/core/Casts.java
type Casts (line 22) | public enum Casts {
FILE: parser-core/src/main/java/nl/basjes/parse/core/Dissector.java
class Dissector (line 62) | public abstract class Dissector implements Serializable {
method initializeFromSettingsParameter (line 75) | public boolean initializeFromSettingsParameter(String settings) {
method dissect (line 85) | public abstract void dissect(Parsable<?> parsable, String inputname)
method getInputType (line 93) | public abstract String getInputType();
method getPossibleOutput (line 101) | public abstract List<String> getPossibleOutput();
method prepareForDissect (line 115) | public abstract EnumSet<Casts> prepareForDissect(String inputname, Str...
method prepareForRun (line 123) | public void prepareForRun() throws InvalidDissectorException {
method getNewInstance (line 135) | public Dissector getNewInstance() throws InvalidDissectorException {
method extractFieldName (line 149) | public String extractFieldName(final String inputname, final String ou...
method initializeNewInstance (line 165) | protected void initializeNewInstance(Dissector newInstance) throws Inv...
method createAdditionalDissectors (line 175) | public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) {
method setInputType (line 179) | public void setInputType(String s) throws InvalidDissectorException {
method toString (line 184) | @Override
FILE: parser-core/src/main/java/nl/basjes/parse/core/Parsable.java
class Parsable (line 28) | public final class Parsable<RECORD> {
method Parsable (line 54) | public Parsable(final Parser<RECORD> parser, final RECORD record, Map<...
method setRootDissection (line 64) | void setRootDissection(final String type, final String value) {
method addDissection (line 77) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 83) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 92) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 98) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 104) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 110) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 119) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 125) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 131) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 137) | public Parsable<RECORD> addDissection(final String base, final String ...
method addDissection (line 142) | private Parsable<RECORD> addDissection(
method getParsableField (line 197) | public ParsedField getParsableField(final String type, final String na...
method getRecord (line 203) | public RECORD getRecord() {
method setAsParsed (line 209) | public void setAsParsed(final ParsedField parsedField) {
method getToBeParsed (line 215) | public Set<ParsedField> getToBeParsed() {
FILE: parser-core/src/main/java/nl/basjes/parse/core/ParsedField.java
class ParsedField (line 19) | public class ParsedField {
method ParsedField (line 25) | public ParsedField(String type, String name, Value value) {
method ParsedField (line 35) | public ParsedField(String type, String name, String value) {
method getType (line 41) | public String getType() {
method getName (line 45) | public String getName() {
method getValue (line 49) | public Value getValue() {
method makeId (line 53) | public static String makeId(String type, String name) {
method getId (line 57) | public String getId() {
method toString (line 61) | public String toString(){
FILE: parser-core/src/main/java/nl/basjes/parse/core/Parser.java
class Parser (line 50) | public class Parser<RECORD> implements Serializable {
type SetterPolicy (line 52) | public enum SetterPolicy {
class DissectorPhase (line 63) | private static class DissectorPhase implements Serializable {
method DissectorPhase (line 64) | DissectorPhase(final String inputType, final String outputType, fina...
method getNeeded (line 106) | public Set<String> getNeeded() {
method getCasts (line 127) | public EnumSet<Casts> getCasts(String name) throws MissingDissectorsEx...
method getAllCasts (line 132) | public Map<String, EnumSet<Casts>> getAllCasts() throws MissingDissect...
method getUsefulIntermediateFields (line 139) | Set<String> getUsefulIntermediateFields() {
method addDissectors (line 145) | public final Parser<RECORD> addDissectors(final List<Dissector> dissec...
method addDissector (line 155) | public final Parser<RECORD> addDissector(final Dissector dissector) {
method dropDissector (line 165) | public final Parser<RECORD> dropDissector(Class<? extends Dissector> d...
method getAllDissectors (line 179) | public final Set<Dissector> getAllDissectors() {
method setRootType (line 185) | public Parser<RECORD> setRootType(final String newRootType) {
method assembleDissectorPhases (line 192) | private void assembleDissectorPhases() throws InvalidDissectorException {
method ignoreMissingDissectors (line 225) | public Parser<RECORD> ignoreMissingDissectors() {
method failOnMissingDissectors (line 233) | public Parser<RECORD> failOnMissingDissectors() {
method assembleDissectors (line 239) | private void assembleDissectors() throws MissingDissectorsException, I...
method findUsefulDissectorsFromField (line 362) | private void findUsefulDissectorsFromField(
method findDissectorInstance (line 462) | private DissectorPhase findDissectorInstance(Set<DissectorPhase> disse...
method getTheMissingFields (line 474) | private Set<String> getTheMissingFields(Set<String> locatedTargets) {
method Parser (line 498) | public Parser(final Class<RECORD> clazz) {
method addParseTarget (line 515) | public Parser<RECORD> addParseTarget(final String setterMethodName,
method addParseTarget (line 523) | public Parser<RECORD> addParseTarget(final String setterMethodName,
method addParseTarget (line 563) | public Parser<RECORD> addParseTarget(final Method method, final String...
method addParseTarget (line 569) | public Parser<RECORD> addParseTarget(final Method method,
method addParseTarget (line 577) | public Parser<RECORD> addParseTarget(final Method method, final List<S...
method addParseTarget (line 583) | public Parser<RECORD> addParseTarget(final Method method,
method setTypeRemappings (line 643) | public Parser<RECORD> setTypeRemappings(Map<String, Set<String>> pType...
method addTypeRemappings (line 652) | public Parser<RECORD> addTypeRemappings(Map<String, Set<String>> addit...
method addTypeRemapping (line 662) | public Parser<RECORD> addTypeRemapping(String input, String newType) {
method addTypeRemapping (line 666) | public Parser<RECORD> addTypeRemapping(String input, String newType, E...
method cleanupFieldValue (line 683) | public static String cleanupFieldValue(String fieldValue) {
method parse (line 702) | public RECORD parse(final String value)
method parse (line 718) | public RECORD parse(final RECORD record, final String value)
method parse (line 728) | Parsable<RECORD> parse(final Parsable<RECORD> parsable)
method store (line 762) | void store(final RECORD record, final String key, final String name, f...
method createParsable (line 882) | private Parsable<RECORD> createParsable(RECORD record) {
method createParsable (line 886) | public Parsable<RECORD> createParsable() {
method getPossiblePaths (line 906) | public List<String> getPossiblePaths() {
method getPossiblePaths (line 916) | public List<String> getPossiblePaths(int maxDepth) {
method getPossiblePaths (line 927) | public List<String> getPossiblePaths(int maxDepth, boolean sortList) {
method findAdditionalPossiblePaths (line 993) | private void findAdditionalPossiblePaths(Map<String, List<String>> pat...
FILE: parser-core/src/main/java/nl/basjes/parse/core/SimpleDissector.java
class SimpleDissector (line 30) | public abstract class SimpleDissector extends Dissector {
method SimpleDissector (line 38) | public SimpleDissector(String inputType, Map<String, EnumSet<Casts>> o...
method getInputType (line 48) | @Override
method setInputType (line 53) | @Override
method getPossibleOutput (line 58) | @Override
method prepareForDissect (line 63) | @Override
method initializeNewInstance (line 69) | @Override
method dissect (line 79) | @Override
method dissect (line 89) | public abstract void dissect(Parsable<?> parsable, String inputname, V...
FILE: parser-core/src/main/java/nl/basjes/parse/core/Value.java
class Value (line 20) | public class Value {
type Filled (line 22) | enum Filled {
method Value (line 33) | public Value(String p) {
method Value (line 38) | public Value(Long p) {
method Value (line 43) | public Value(Double p) {
method getString (line 48) | public String getString() {
method getLong (line 59) | public Long getLong() {
method getDouble (line 74) | public Double getDouble() {
method toString (line 89) | @Override
FILE: parser-core/src/main/java/nl/basjes/parse/core/exceptions/DissectionFailure.java
class DissectionFailure (line 19) | public class DissectionFailure extends Exception {
method DissectionFailure (line 22) | public DissectionFailure(String message) {
method DissectionFailure (line 25) | public DissectionFailure(String message, Throwable cause) {
FILE: parser-core/src/main/java/nl/basjes/parse/core/exceptions/FatalErrorDuringCallOfSetterMethod.java
class FatalErrorDuringCallOfSetterMethod (line 19) | public class FatalErrorDuringCallOfSetterMethod extends RuntimeException {
method FatalErrorDuringCallOfSetterMethod (line 22) | public FatalErrorDuringCallOfSetterMethod(String message){
method FatalErrorDuringCallOfSetterMethod (line 26) | public FatalErrorDuringCallOfSetterMethod(String message, Throwable ca...
FILE: parser-core/src/main/java/nl/basjes/parse/core/exceptions/InvalidDissectorException.java
class InvalidDissectorException (line 19) | public class InvalidDissectorException extends Exception {
method InvalidDissectorException (line 22) | public InvalidDissectorException() {
method InvalidDissectorException (line 26) | public InvalidDissectorException(String message) {
method InvalidDissectorException (line 30) | public InvalidDissectorException(String message, Throwable cause) {
FILE: parser-core/src/main/java/nl/basjes/parse/core/exceptions/InvalidFieldMethodSignature.java
class InvalidFieldMethodSignature (line 21) | public class InvalidFieldMethodSignature extends RuntimeException {
method InvalidFieldMethodSignature (line 24) | public InvalidFieldMethodSignature(final Method method) {
FILE: parser-core/src/main/java/nl/basjes/parse/core/exceptions/MissingDissectorsException.java
class MissingDissectorsException (line 19) | public class MissingDissectorsException extends Exception {
method MissingDissectorsException (line 22) | public MissingDissectorsException(String message) {
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserCastsTest.java
class ParserCastsTest (line 38) | class ParserCastsTest {
class MyDissector (line 40) | public static class MyDissector extends Dissector {
method MyDissector (line 42) | public MyDissector() {
method dissect (line 46) | @Override
method getInputType (line 68) | @Override
method getPossibleOutput (line 73) | @Override
method prepareForDissect (line 114) | @Override
class MyParser (line 120) | public static class MyParser<RECORD> extends Parser<RECORD> {
method MyParser (line 121) | public MyParser(final Class<RECORD> clazz) {
class MyRecord (line 128) | public static class MyRecord {
method setStringNull (line 130) | @Field({"OUTPUT_TYPE:string_null",
method setStringGood (line 139) | @Field({"OUTPUT_TYPE:string_good",
method setLongNull (line 148) | @Field({"OUTPUT_TYPE:long_null",
method setLongGood (line 157) | @Field({"OUTPUT_TYPE:long_good",
method setDoubleNull (line 165) | @Field({"OUTPUT_TYPE:double_null",
method setDoubleGood (line 174) | @Field({"OUTPUT_TYPE:double_good",
method setLongWrongSignature (line 182) | @SuppressWarnings("UnusedParameters")
method setDoubleWrongSignature (line 192) | @SuppressWarnings("UnusedParameters")
method testValidCasting (line 203) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserDissectionOutputTypesTest.java
class ParserDissectionOutputTypesTest (line 34) | class ParserDissectionOutputTypesTest {
class TestDissector (line 36) | public static class TestDissector extends Dissector {
method TestDissector (line 38) | public TestDissector() {
method dissect (line 42) | @Override
method getInputType (line 67) | @Override
method getPossibleOutput (line 72) | @Override
method prepareForDissect (line 117) | @Override
class TestParser (line 123) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 124) | TestParser(final Class<RECORD> clazz) {
class TestRecord (line 131) | @SuppressWarnings("unused")
method setStringNull (line 135) | @Field({
method setLongNull (line 148) | @Field({
method setDoubleNull (line 160) | @Field({
method setString (line 170) | @Field({
method setStringFromDouble (line 184) | @Field({
method setLong (line 193) | @Field({
method setDouble (line 208) | @Field({
method testSetterTypes (line 221) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserDuplicateOutputTest.java
class ParserDuplicateOutputTest (line 28) | class ParserDuplicateOutputTest {
class MyDissector (line 30) | public abstract static class MyDissector extends SimpleDissector {
method MyDissector (line 36) | public MyDissector() {
class FooDissector (line 41) | public static class FooDissector extends MyDissector {
method dissect (line 42) | @Override
class BarDissector (line 47) | public static class BarDissector extends MyDissector {
method dissect (line 48) | @Override
method testParseString (line 55) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserExceptionsTest.java
class ParserExceptionsTest (line 36) | class ParserExceptionsTest {
class TestDissector (line 38) | public static class TestDissector extends Dissector {
method TestDissector (line 43) | public TestDissector(String inputType, String outputType, String out...
method init (line 47) | public final void init(String inputtype, String outputtype, String o...
method initializeNewInstance (line 53) | protected void initializeNewInstance(Dissector newInstance) {
method dissect (line 57) | @Override
method getInputType (line 63) | @Override
method getPossibleOutput (line 68) | @Override
method prepareForDissect (line 75) | @Override
class TestDissectorOne (line 81) | public static class TestDissectorOne extends TestDissector {
method TestDissectorOne (line 82) | public TestDissectorOne() {
class TestDissectorTwo (line 87) | public static class TestDissectorTwo extends TestDissector {
method TestDissectorTwo (line 88) | public TestDissectorTwo() {
class TestDissectorThree (line 93) | public static class TestDissectorThree extends TestDissector {
method TestDissectorThree (line 94) | public TestDissectorThree() {
class TestDissectorFour (line 99) | public static class TestDissectorFour extends TestDissector {
method TestDissectorFour (line 100) | public TestDissectorFour() {
class TestParser (line 105) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 106) | public TestParser(final Class<RECORD> clazz) {
class TestRecord (line 120) | @SuppressWarnings("unused")
method setValue1 (line 123) | @Field("SOMETYPE:output1")
method setValue2 (line 130) | public void setValue2(String name, String value) {
method setValue3 (line 137) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2" })
method setValue4 (line 149) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2", "SOMETYPE:output1"...
method setValue5 (line 161) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2", "SOMETYPE:*", "OTH...
method setValue6 (line 171) | @Field({ "FOO:output1.foo"})
method setValue7 (line 177) | @Field({ "BAR:output1.bar"})
method setValue8 (line 183) | public void setValue8(String name, String value) {
method badSetter1 (line 187) | @SuppressWarnings({"UnusedDeclaration", "EmptyMethod"})
method badSetter2 (line 191) | @SuppressWarnings({"UnusedDeclaration", "EmptyMethod"})
method testParseString (line 196) | @Test
method testGetPossiblePaths (line 217) | @Test
method testBadSetter1 (line 232) | @Test
method testBadSetter2 (line 242) | @Test
class BrokenTestDissector (line 252) | public static class BrokenTestDissector extends Dissector {
method BrokenTestDissector (line 254) | public BrokenTestDissector() {
method initializeFromSettingsParameter (line 257) | @Override
method dissect (line 262) | @Override
method getInputType (line 266) | @Override
method getPossibleOutput (line 271) | @Override
method prepareForDissect (line 278) | @Override
method prepareForRun (line 283) | @Override
method testBrokenDissector (line 289) | @Test
class BrokenTestDissector2 (line 301) | public static class BrokenTestDissector2 extends BrokenTestDissector {
method BrokenTestDissector2 (line 303) | BrokenTestDissector2() {
method testBrokenDissector2 (line 307) | @Test
method testChangeAfterStart (line 319) | @Test
method testDropDissector1 (line 326) | @Test
method testDropDissector2 (line 336) | @Test
method testDropDissector3 (line 345) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserInfiniteLoopTest.java
class ParserInfiniteLoopTest (line 28) | class ParserInfiniteLoopTest {
class TestDissector (line 30) | public static class TestDissector extends Dissector {
method TestDissector (line 32) | public TestDissector() {
method dissect (line 36) | @Override
method getInputType (line 41) | @Override
method getPossibleOutput (line 46) | @Override
method prepareForDissect (line 53) | @Override
class TestParser (line 59) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 60) | public TestParser(final Class<RECORD> clazz) {
class TestRecord (line 67) | public static class TestRecord {
method set (line 68) | @SuppressWarnings({"EmptyMethod", "UnusedParameters"})
method testInfiniteRecursionAvoidance (line 82) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserNormalTest.java
class ParserNormalTest (line 40) | class ParserNormalTest {
class MyDissector (line 42) | public static class MyDissector extends Dissector {
method MyDissector (line 48) | public MyDissector(String inputType, String outputType, String outpu...
method init (line 55) | public void init(String inputtype, String outputtype, String outputn...
method initializeNewInstance (line 62) | @Override
method dissect (line 67) | @Override
method getInputType (line 75) | @Override
method getPossibleOutput (line 80) | @Override
method prepareForDissect (line 87) | @Override
class MyDissectorOne (line 99) | public static class MyDissectorOne extends MyDissector {
method MyDissectorOne (line 100) | public MyDissectorOne() {
class MyDissectorTwo (line 105) | public static class MyDissectorTwo extends MyDissector {
method MyDissectorTwo (line 106) | public MyDissectorTwo() {
class MyDissectorThree (line 111) | public static class MyDissectorThree extends MyDissector {
method MyDissectorThree (line 112) | public MyDissectorThree() {
class MyDissectorFour (line 117) | public static class MyDissectorFour extends MyDissector {
method MyDissectorFour (line 118) | public MyDissectorFour() {
class MyDissectorWildCard (line 123) | public static class MyDissectorWildCard extends MyDissector {
method MyDissectorWildCard (line 124) | public MyDissectorWildCard() {
class TestParser (line 130) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 131) | public TestParser(final Class<RECORD> clazz) {
method testParseString (line 142) | @Test
method testParseStringInstantiate (line 168) | @Test
method testMissingDissector (line 192) | @Test
method testGetPossiblePaths (line 206) | @Test
method testAddTypeRemapping (line 220) | @Test
method testAddTypeRemappings (line 234) | @Test
method testSetTypeRemapping (line 248) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserNormalTestRecord.java
class ParserNormalTestRecord (line 19) | public class ParserNormalTestRecord {
method ParserNormalTestRecord (line 21) | public ParserNormalTestRecord() {
method setValue1 (line 27) | @Field("SOMETYPE:output1")
method getOutput1 (line 32) | public String getOutput1() {
method setValue2 (line 39) | public void setValue2(String name, String value) {
method getOutput2 (line 43) | public String getOutput2() {
method setValue3 (line 50) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2" })
method getOutput3a (line 59) | public String getOutput3a() {
method getOutput3b (line 63) | public String getOutput3b() {
method setValue4 (line 70) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2", "SOMETYPE:output1", ...
method getOutput4a (line 79) | public String getOutput4a() {
method getOutput4b (line 83) | public String getOutput4b() {
method setValue5 (line 90) | @Field({ "SOMETYPE:output1", "OTHERTYPE:output2", "SOMETYPE:*", "OTHER...
method getOutput5a (line 99) | public String getOutput5a() {
method getOutput5b (line 103) | public String getOutput5b() {
method setValue6 (line 109) | @Field({ "FOO:output1.foo" })
method getOutput6 (line 114) | public String getOutput6() {
method setValue7 (line 120) | @Field({ "BAR:output1.bar" })
method getOutput7 (line 125) | public String getOutput7() {
method setValue8 (line 131) | @Field({ "WILD:output1.wild" })
method getOutput8 (line 136) | public String getOutput8() {
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserResetTest.java
class ParserResetTest (line 34) | class ParserResetTest {
class WildCardDissector (line 36) | public static class WildCardDissector extends SimpleDissector {
method WildCardDissector (line 45) | public WildCardDissector() {
method dissect (line 49) | @Override
method prepareForDissect (line 56) | @Override
class DuplicateTestRecord (line 68) | public static class DuplicateTestRecord {
method setStringValue (line 71) | public void setStringValue(final String name, final String value) {
method getStringValues (line 74) | public List<String> getStringValues(final String name) {
method testParserReset (line 80) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserTypeColissionTest.java
class ParserTypeColissionTest (line 29) | public class ParserTypeColissionTest {
class TestDissector (line 31) | public static class TestDissector extends Dissector {
method TestDissector (line 37) | public TestDissector(String inputType, String outputType, String out...
method init (line 44) | public void init(String inputtype, String outputtype, String outputn...
method initializeNewInstance (line 51) | @Override
method dissect (line 56) | @Override
method getInputType (line 63) | @Override
method getPossibleOutput (line 68) | @Override
method prepareForDissect (line 75) | @Override
class TestDissectorOne (line 81) | public static class TestDissectorOne extends TestDissector {
method TestDissectorOne (line 82) | public TestDissectorOne() {
class TestDissectorTwo (line 87) | public static class TestDissectorTwo extends TestDissector {
method TestDissectorTwo (line 88) | public TestDissectorTwo() {
class TestDissectorSubOne (line 93) | public static class TestDissectorSubOne extends TestDissector {
method TestDissectorSubOne (line 94) | public TestDissectorSubOne() {
class TestDissectorSubTwo (line 99) | public static class TestDissectorSubTwo extends TestDissector {
method TestDissectorSubTwo (line 100) | public TestDissectorSubTwo() {
class TestDissectorSubSubOne (line 105) | public static class TestDissectorSubSubOne extends TestDissector {
method TestDissectorSubSubOne (line 106) | public TestDissectorSubSubOne() {
class TestDissectorSubSubTwo (line 111) | public static class TestDissectorSubSubTwo extends TestDissector {
method TestDissectorSubSubTwo (line 112) | public TestDissectorSubSubTwo() {
class TestParser (line 117) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 118) | public TestParser(final Class<RECORD> clazz) {
class TestRecord (line 130) | public static class TestRecord {
method setValue1 (line 133) | @Field("SOMETYPE:output")
method setValue2 (line 140) | @Field("OTHERTYPE:output")
method setValue3 (line 147) | @Field("SOMESUBSUBTYPE:output.output.output")
method setValue4 (line 154) | @Field("OTHERSUBSUBTYPE:output.output.output")
method testParseString (line 161) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/ParserTypeRemappingEdgeCase.java
class ParserTypeRemappingEdgeCase (line 30) | public class ParserTypeRemappingEdgeCase {
class TestDissectorLongAsString (line 32) | public static class TestDissectorLongAsString extends SimpleDissector {
method TestDissectorLongAsString (line 39) | public TestDissectorLongAsString() {
method dissect (line 43) | @Override
class TestParser (line 50) | public static class TestParser<RECORD> extends Parser<RECORD> {
method TestParser (line 51) | public TestParser(final Class<RECORD> clazz) {
class Record (line 58) | public static class Record {
method set1 (line 61) | public void set1(String name, String value){
method set2 (line 68) | public void set2(String name, String value){
method set (line 75) | public void set(String name, Long value){
method testParseString (line 81) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/TestBadAPIUsage.java
class TestBadAPIUsage (line 38) | public class TestBadAPIUsage {
method testChangingInputTypeShouldNotBePossibleByDefault (line 40) | @Test
method testDissectorString (line 47) | @Test
class NullInputDissector (line 55) | public static class NullInputDissector extends Dissector {
method dissect (line 56) | @Override
method getInputType (line 59) | @Override
method getPossibleOutput (line 63) | @Override
method prepareForDissect (line 67) | @Override
method testNullInputHandling (line 73) | @Test
class NullOutputDissector (line 80) | public static class NullOutputDissector extends Dissector {
method dissect (line 81) | @Override
method getInputType (line 84) | @Override
method getPossibleOutput (line 88) | @Override
method prepareForDissect (line 92) | @Override
method testNullOutputHandling (line 98) | @Test
class EmptyOutputDissector (line 105) | public static class EmptyOutputDissector extends Dissector {
method dissect (line 106) | @Override
method getInputType (line 109) | @Override
method getPossibleOutput (line 113) | @Override
method prepareForDissect (line 117) | @Override
method testEmptyOutputHandling (line 123) | @Test
method testFailZeroDissectors (line 130) | @Test
method testFailOnMissingDissectors (line 142) | @Test
method testIgnoreMissingDissectors (line 157) | @Test
method testNoSuchSetter (line 170) | @Test
method testBadParameters (line 184) | @Test
method testFieldCleanup (line 199) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSetters.java
class TestFieldSetters (line 36) | class TestFieldSetters {
class TestFieldSettersRecord (line 38) | public static class TestFieldSettersRecord extends TestRecord {
method setS (line 39) | private void setS(String prefix, String name, String value) {
method setL (line 42) | private void setL(String prefix, String name, Long value) {
method setD (line 45) | private void setD(String prefix, String name, Double value) {
method setAD (line 49) | @Field(value = "ANY:any" ) public ...
method setSD (line 50) | @Field(value = "STRING:string" ) public ...
method setID (line 51) | @Field(value = "INT:int" ) public ...
method setLD (line 52) | @Field(value = "LONG:long" ) public ...
method setFD (line 53) | @Field(value = "FLOAT:float" ) public ...
method setDD (line 54) | @Field(value = "DOUBLE:double" ) public ...
method setAD (line 55) | @Field(value = "ANY:any" ) public ...
method setSD (line 56) | @Field(value = "STRING:string" ) public ...
method setID (line 57) | @Field(value = "INT:int" ) public ...
method setLD (line 58) | @Field(value = "LONG:long" ) public ...
method setFD (line 59) | @Field(value = "FLOAT:float" ) public ...
method setDD (line 60) | @Field(value = "DOUBLE:double" ) public ...
method setAD (line 61) | @Field(value = "ANY:any" ) public ...
method setSD (line 62) | @Field(value = "STRING:string" ) public ...
method setID (line 63) | @Field(value = "INT:int" ) public ...
method setLD (line 64) | @Field(value = "LONG:long" ) public ...
method setFD (line 65) | @Field(value = "FLOAT:float" ) public ...
method setDD (line 66) | @Field(value = "DOUBLE:double" ) public ...
method setAA (line 68) | @Field(value = "ANY:any", setterPolicy = ALWAYS ) public ...
method setSA (line 69) | @Field(value = "STRING:string", setterPolicy = ALWAYS ) public ...
method setIA (line 70) | @Field(value = "INT:int", setterPolicy = ALWAYS ) public ...
method setLA (line 71) | @Field(value = "LONG:long", setterPolicy = ALWAYS ) public ...
method setFA (line 72) | @Field(value = "FLOAT:float", setterPolicy = ALWAYS ) public ...
method setDA (line 73) | @Field(value = "DOUBLE:double", setterPolicy = ALWAYS ) public ...
method setAA (line 74) | @Field(value = "ANY:any", setterPolicy = ALWAYS ) public ...
method setSA (line 75) | @Field(value = "STRING:string", setterPolicy = ALWAYS ) public ...
method setIA (line 76) | @Field(value = "INT:int", setterPolicy = ALWAYS ) public ...
method setLA (line 77) | @Field(value = "LONG:long", setterPolicy = ALWAYS ) public ...
method setFA (line 78) | @Field(value = "FLOAT:float", setterPolicy = ALWAYS ) public ...
method setDA (line 79) | @Field(value = "DOUBLE:double", setterPolicy = ALWAYS ) public ...
method setAA (line 80) | @Field(value = "ANY:any", setterPolicy = ALWAYS ) public ...
method setSA (line 81) | @Field(value = "STRING:string", setterPolicy = ALWAYS ) public ...
method setIA (line 82) | @Field(value = "INT:int", setterPolicy = ALWAYS ) public ...
method setLA (line 83) | @Field(value = "LONG:long", setterPolicy = ALWAYS ) public ...
method setFA (line 84) | @Field(value = "FLOAT:float", setterPolicy = ALWAYS ) public ...
method setDA (line 85) | @Field(value = "DOUBLE:double", setterPolicy = ALWAYS ) public ...
method setAN (line 87) | @Field(value = "ANY:any", setterPolicy = NOT_NULL ) public ...
method setSN (line 88) | @Field(value = "STRING:string", setterPolicy = NOT_NULL ) public ...
method setIN (line 89) | @Field(value = "INT:int", setterPolicy = NOT_NULL ) public ...
method setLN (line 90) | @Field(value = "LONG:long", setterPolicy = NOT_NULL ) public ...
method setFN (line 91) | @Field(value = "FLOAT:float", setterPolicy = NOT_NULL ) public ...
method setDN (line 92) | @Field(value = "DOUBLE:double", setterPolicy = NOT_NULL ) public ...
method setAN (line 93) | @Field(value = "ANY:any", setterPolicy = NOT_NULL ) public ...
method setSN (line 94) | @Field(value = "STRING:string", setterPolicy = NOT_NULL ) public ...
method setIN (line 95) | @Field(value = "INT:int", setterPolicy = NOT_NULL ) public ...
method setLN (line 96) | @Field(value = "LONG:long", setterPolicy = NOT_NULL ) public ...
method setFN (line 97) | @Field(value = "FLOAT:float", setterPolicy = NOT_NULL ) public ...
method setDN (line 98) | @Field(value = "DOUBLE:double", setterPolicy = NOT_NULL ) public ...
method setAN (line 99) | @Field(value = "ANY:any", setterPolicy = NOT_NULL ) public ...
method setSN (line 100) | @Field(value = "STRING:string", setterPolicy = NOT_NULL ) public ...
method setIN (line 101) | @Field(value = "INT:int", setterPolicy = NOT_NULL ) public ...
method setLN (line 102) | @Field(value = "LONG:long", setterPolicy = NOT_NULL ) public ...
method setFN (line 103) | @Field(value = "FLOAT:float", setterPolicy = NOT_NULL ) public ...
method setDN (line 104) | @Field(value = "DOUBLE:double", setterPolicy = NOT_NULL ) public ...
method setAE (line 106) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public ...
method setSE (line 107) | @Field(value = "STRING:string", setterPolicy = NOT_EMPTY) public ...
method setIE (line 108) | @Field(value = "INT:int", setterPolicy = NOT_EMPTY) public ...
method setLE (line 109) | @Field(value = "LONG:long", setterPolicy = NOT_EMPTY) public ...
method setFE (line 110) | @Field(value = "FLOAT:float", setterPolicy = NOT_EMPTY) public ...
method setDE (line 111) | @Field(value = "DOUBLE:double", setterPolicy = NOT_EMPTY) public ...
method setAE (line 112) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public ...
method setSE (line 113) | @Field(value = "STRING:string", setterPolicy = NOT_EMPTY) public ...
method setIE (line 114) | @Field(value = "INT:int", setterPolicy = NOT_EMPTY) public ...
method setLE (line 115) | @Field(value = "LONG:long", setterPolicy = NOT_EMPTY) public ...
method setFE (line 116) | @Field(value = "FLOAT:float", setterPolicy = NOT_EMPTY) public ...
method setDE (line 117) | @Field(value = "DOUBLE:double", setterPolicy = NOT_EMPTY) public ...
method setAE (line 118) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public ...
method setSE (line 119) | @Field(value = "STRING:string", setterPolicy = NOT_EMPTY) public ...
method setIE (line 120) | @Field(value = "INT:int", setterPolicy = NOT_EMPTY) public ...
method setLE (line 121) | @Field(value = "LONG:long", setterPolicy = NOT_EMPTY) public ...
method setFE (line 122) | @Field(value = "FLOAT:float", setterPolicy = NOT_EMPTY) public ...
method setDE (line 123) | @Field(value = "DOUBLE:double", setterPolicy = NOT_EMPTY) public ...
method testNormalValues (line 126) | @Test
method testEmptyValues (line 214) | @Test
method testNullValues (line 304) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersAlwaysCombined.java
class TestFieldSettersAlwaysCombined (line 30) | class TestFieldSettersAlwaysCombined {
class TestRecordString (line 32) | public static class TestRecordString extends TestRecord {
method set (line 33) | @Field(value = {
class TestRecordLong (line 46) | public static class TestRecordLong extends TestRecord {
method set (line 47) | @Field(value = {
class TestRecordDouble (line 57) | public static class TestRecordDouble extends TestRecord {
method set (line 58) | @Field(value = {
method testString (line 69) | @Test
method testLong (line 84) | @Test
method testDouble (line 96) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersAlwaysSeparate.java
class TestFieldSettersAlwaysSeparate (line 30) | public class TestFieldSettersAlwaysSeparate {
class TestRecordString (line 32) | public static class TestRecordString extends TestRecord {
method set (line 33) | private void set(String name, String value) {
method setA (line 38) | @Field(value = "ANY:any", setterPolicy = ALWAYS) public void s...
method setS (line 39) | @Field(value = "STRING:string", setterPolicy = ALWAYS) public void s...
method setI (line 40) | @Field(value = "INT:int", setterPolicy = ALWAYS) public void s...
method setL (line 41) | @Field(value = "LONG:long", setterPolicy = ALWAYS) public void s...
method setF (line 42) | @Field(value = "FLOAT:float", setterPolicy = ALWAYS) public void s...
method setD (line 43) | @Field(value = "DOUBLE:double", setterPolicy = ALWAYS) public void s...
class TestRecordLong (line 47) | public static class TestRecordLong extends TestRecord {
method set (line 48) | private void set(String name, Long value) {
method setA (line 53) | @Field(value = "ANY:any", setterPolicy = ALWAYS) public void s...
method setI (line 54) | @Field(value = "INT:int", setterPolicy = ALWAYS) public void s...
method setL (line 55) | @Field(value = "LONG:long", setterPolicy = ALWAYS) public void s...
class TestRecordDouble (line 59) | public static class TestRecordDouble extends TestRecord {
method set (line 60) | private void set(String name, Double value) {
method setA (line 65) | @Field(value = "ANY:any", setterPolicy = ALWAYS) public void s...
method setF (line 66) | @Field(value = "FLOAT:float", setterPolicy = ALWAYS) public void s...
method setD (line 67) | @Field(value = "DOUBLE:double", setterPolicy = ALWAYS) public void s...
method testString (line 71) | @Test
method testLong (line 86) | @Test
method testDouble (line 98) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersNotEmpty.java
class TestFieldSettersNotEmpty (line 30) | public class TestFieldSettersNotEmpty {
class TestRecordString (line 32) | public static class TestRecordString extends TestRecord {
method set (line 33) | @Field(value = {
method setA (line 46) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public voi...
method setS (line 47) | @Field(value = "STRING:string", setterPolicy = NOT_EMPTY) public voi...
method setI (line 48) | @Field(value = "INT:int", setterPolicy = NOT_EMPTY) public voi...
method setL (line 49) | @Field(value = "LONG:long", setterPolicy = NOT_EMPTY) public voi...
method setF (line 50) | @Field(value = "FLOAT:float", setterPolicy = NOT_EMPTY) public voi...
method setD (line 51) | @Field(value = "DOUBLE:double", setterPolicy = NOT_EMPTY) public voi...
class TestRecordLong (line 55) | public static class TestRecordLong extends TestRecord {
method set (line 56) | @Field(value = {
method setA (line 66) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public voi...
method setI (line 67) | @Field(value = "INT:int", setterPolicy = NOT_EMPTY) public voi...
method setL (line 68) | @Field(value = "LONG:long", setterPolicy = NOT_EMPTY) public voi...
class TestRecordDouble (line 72) | public static class TestRecordDouble extends TestRecord {
method set (line 73) | @Field(value = {
method setA (line 83) | @Field(value = "ANY:any", setterPolicy = NOT_EMPTY) public voi...
method setF (line 84) | @Field(value = "FLOAT:float", setterPolicy = NOT_EMPTY) public voi...
method setD (line 85) | @Field(value = "DOUBLE:double", setterPolicy = NOT_EMPTY) public voi...
method testString (line 89) | @Test
method testLong (line 104) | @Test
method testDouble (line 116) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersNotNull.java
class TestFieldSettersNotNull (line 30) | public class TestFieldSettersNotNull {
class TestRecordString (line 32) | public static class TestRecordString extends TestRecord {
method set (line 33) | @Field(value = {
method setA (line 46) | @Field(value = "ANY:any", setterPolicy = NOT_NULL) public void...
method setS (line 47) | @Field(value = "STRING:string", setterPolicy = NOT_NULL) public void...
method setI (line 48) | @Field(value = "INT:int", setterPolicy = NOT_NULL) public void...
method setL (line 49) | @Field(value = "LONG:long", setterPolicy = NOT_NULL) public void...
method setF (line 50) | @Field(value = "FLOAT:float", setterPolicy = NOT_NULL) public void...
method setD (line 51) | @Field(value = "DOUBLE:double", setterPolicy = NOT_NULL) public void...
class TestRecordLong (line 55) | public static class TestRecordLong extends TestRecord {
method set (line 56) | @Field(value = {
method setA (line 66) | @Field(value = "ANY:any", setterPolicy = NOT_NULL) public void...
method setI (line 67) | @Field(value = "INT:int", setterPolicy = NOT_NULL) public void...
method setL (line 68) | @Field(value = "LONG:long", setterPolicy = NOT_NULL) public void...
class TestRecordDouble (line 72) | public static class TestRecordDouble extends TestRecord {
method set (line 73) | @Field(value = {
method setA (line 83) | @Field(value = "ANY:any", setterPolicy = NOT_NULL) public void...
method setF (line 84) | @Field(value = "FLOAT:float", setterPolicy = NOT_NULL) public void...
method setD (line 85) | @Field(value = "DOUBLE:double", setterPolicy = NOT_NULL) public void...
method testString (line 89) | @Test
method testLong (line 104) | @Test
method testDouble (line 116) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/convert/ValueConvertTest.java
class ValueConvertTest (line 36) | public class ValueConvertTest {
method verifyTypeConversionStoM (line 38) | @Test
method verifyTypeConversionMtoS (line 50) | @Test
method verifyTypeConversionPossibleFields (line 62) | @Test
class TypeConvertBaseDissector (line 74) | public abstract static class TypeConvertBaseDissector extends SimpleDi...
method fillOutputConfig (line 78) | private static HashMap<String, EnumSet<Casts>> fillOutputConfig(Stri...
method TypeConvertBaseDissector (line 84) | public TypeConvertBaseDissector(String nInputType, String nOutputTyp...
method initializeNewInstance (line 90) | @Override
class SecondsToMilliseconds (line 98) | public static class SecondsToMilliseconds extends TypeConvertBaseDisse...
method SecondsToMilliseconds (line 100) | public SecondsToMilliseconds() {
method dissect (line 104) | @Override
class MillisecondsToSeconds (line 110) | public static class MillisecondsToSeconds extends TypeConvertBaseDisse...
method MillisecondsToSeconds (line 111) | public MillisecondsToSeconds() {
method dissect (line 115) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/reference/BarDissector.java
class BarDissector (line 33) | public class BarDissector extends SimpleDissector {
method BarDissector (line 45) | public BarDissector() {
method dissect (line 49) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/reference/FooDissector.java
class FooDissector (line 33) | public class FooDissector extends SimpleDissector {
method FooDissector (line 45) | public FooDissector() {
method dissect (line 49) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/reference/FooSpecialDissector.java
class FooSpecialDissector (line 21) | public class FooSpecialDissector extends FooDissector {
method createAdditionalDissectors (line 22) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/reference/ReferenceTest.java
class ReferenceTest (line 24) | public class ReferenceTest {
method verifyFoo (line 26) | @Test
method verifyBar (line 53) | @Test
method runManuallyCombined (line 80) | @Test
method runAutomaticallyAddedBar (line 133) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/reference/ReferenceTestDouble.java
class ReferenceTestDouble (line 35) | public class ReferenceTestDouble {
method verifyRemap (line 37) | @Test
method verifyFooInput (line 47) | @Test
method verifyBarInput (line 74) | @Test
method runDoubleDissectors (line 101) | @Test
class InputCreatingDissector (line 156) | public static class InputCreatingDissector extends Dissector {
method dissect (line 157) | @Override
method getInputType (line 162) | @Override
method getPossibleOutput (line 167) | @Override
method prepareForDissect (line 172) | @Override
class RemapInputDissector (line 179) | public static class RemapInputDissector extends Dissector {
method dissect (line 180) | @Override
method getInputType (line 185) | @Override
method getPossibleOutput (line 190) | @Override
method prepareForDissect (line 195) | @Override
class FooInputDissector (line 202) | public static class FooInputDissector extends FooDissector {
method getInputType (line 203) | @Override
class BarInputDissector (line 209) | public static class BarInputDissector extends BarDissector {
method getInputType (line 210) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/DissectorTester.java
class DissectorTester (line 49) | public final class DissectorTester implements Serializable {
method DissectorTester (line 66) | private DissectorTester() {
method create (line 69) | public static DissectorTester create() {
method withParser (line 73) | public DissectorTester withParser(Parser<TestRecord> newParser) {
method withDissector (line 85) | public DissectorTester withDissector(String fieldName, Dissector disse...
method withDissector (line 90) | public DissectorTester withDissector(Dissector dissector) {
method withInput (line 98) | public DissectorTester withInput(String inputValue) {
method addStringSetter (line 103) | private void addStringSetter(String fieldname) {
method addLongSetter (line 112) | private void addLongSetter(String fieldname) {
method addDoubleSetter (line 121) | private void addDoubleSetter(String fieldname) {
method expect (line 130) | public DissectorTester expect(String fieldname, String expected) {
method expect (line 137) | public DissectorTester expect(String fieldname, Long expected) {
method expect (line 144) | public DissectorTester expect(String fieldname, Integer expected) {
method expect (line 148) | public DissectorTester expect(String fieldname, Double expected) {
method expect (line 155) | public DissectorTester expect(String fieldname, Float expected) {
method expectNull (line 159) | public DissectorTester expectNull(String fieldname) {
method expectValuePresent (line 166) | public DissectorTester expectValuePresent(String fieldname) {
method expectAbsentString (line 177) | public DissectorTester expectAbsentString(String fieldname) {
method expectAbsentLong (line 184) | public DissectorTester expectAbsentLong(String fieldname) {
method expectAbsentDouble (line 191) | public DissectorTester expectAbsentDouble(String fieldname) {
method expectPossible (line 198) | public DissectorTester expectPossible(String fieldname) {
method verbose (line 204) | public DissectorTester verbose() {
method withPathPrefix (line 211) | public DissectorTester withPathPrefix(String prefix) {
method addPrefix (line 220) | String addPrefix(String field) {
class ExpectationResult (line 227) | private static class ExpectationResult {
method ExpectationResult (line 233) | ExpectationResult(String expectation, String field, Object value, St...
method expectEquals (line 245) | private void expectEquals(List<ExpectationResult> expectationResults, ...
method checkExpectations (line 267) | public DissectorTester checkExpectations() {
method checkExpectationsDirect (line 276) | private DissectorTester checkExpectationsDirect() {
method summarizeResults (line 301) | private void summarizeResults(List<ExpectationResult> results) {
method checkExpectedValues (line 373) | private List<ExpectationResult> checkExpectedValues() {
method checkExpectedAbsent (line 462) | private List<ExpectationResult> checkExpectedAbsent() {
method parse (line 523) | private TestRecord parse(String inputValue) {
method checkExpectedPossible (line 536) | private List<ExpectationResult> checkExpectedPossible() {
method checkDissectors (line 558) | private List<ExpectationResult> checkDissectors() {
method centerPadding (line 587) | private String centerPadding(String name, char center, int longestLeft...
method padding (line 591) | private String padding(String name, int longestFieldName) {
method padding (line 595) | private String padding(String name, int longestFieldName, char pad) {
method printDissectors (line 608) | public DissectorTester printDissectors() {
method printPossible (line 625) | public DissectorTester printPossible() {
method getPossible (line 636) | public List<String> getPossible() {
method printAllPossibleValues (line 640) | public DissectorTester printAllPossibleValues() {
method printSeparator (line 718) | public DissectorTester printSeparator() {
class DummyDissector (line 725) | public static class DummyDissector extends Dissector {
method DummyDissector (line 730) | public DummyDissector() {
method DummyDissector (line 733) | public DummyDissector(String newOutputType, String newFieldName) {
method dissect (line 738) | @Override
method getInputType (line 744) | @Override
method getPossibleOutput (line 749) | @Override
method prepareForDissect (line 754) | @Override
method initializeNewInstance (line 759) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/EmptyValuesDissector.java
class EmptyValuesDissector (line 25) | public class EmptyValuesDissector extends UltimateDummyDissector {
method EmptyValuesDissector (line 28) | public EmptyValuesDissector() {
method EmptyValuesDissector (line 31) | public EmptyValuesDissector(String inputType) {
method dissect (line 35) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/MyDissectorTester.java
class MyDissectorTester (line 24) | public class MyDissectorTester {
method testPrefixInserter (line 25) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/NormalValuesDissector.java
class NormalValuesDissector (line 25) | public class NormalValuesDissector extends UltimateDummyDissector {
method NormalValuesDissector (line 28) | public NormalValuesDissector() {
method NormalValuesDissector (line 31) | public NormalValuesDissector(String inputType) {
method dissect (line 35) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/NullValuesDissector.java
class NullValuesDissector (line 25) | public class NullValuesDissector extends UltimateDummyDissector {
method NullValuesDissector (line 28) | public NullValuesDissector() {
method NullValuesDissector (line 31) | public NullValuesDissector(String inputType) {
method dissect (line 35) | @Override
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/TestRecord.java
class TestRecord (line 34) | public class TestRecord {
method setVerbose (line 42) | public void setVerbose() {
method getAllNames (line 48) | public Set<String> getAllNames() {
method setStringValue (line 56) | public void setStringValue(final String name, final String value) {
method setLongValue (line 63) | public void setLongValue(final String name, final Long value) {
method setDoubleValue (line 70) | public void setDoubleValue(final String name, final Double value) {
method getStringValue (line 77) | public String getStringValue(final String name) {
method getLongValue (line 85) | public Long getLongValue(final String name) {
method getDoubleValue (line 93) | public Double getDoubleValue(final String name) {
method getStringValues (line 101) | public Set<String> getStringValues(final String name) {
method getLongValues (line 105) | public Set<Long> getLongValues(final String name) {
method getDoubleValues (line 109) | public Set<Double> getDoubleValues(final String name) {
method hasStringValue (line 113) | public boolean hasStringValue(final String name) {
method hasLongValue (line 117) | public boolean hasLongValue(final String name) {
method hasDoubleValue (line 121) | public boolean hasDoubleValue(final String name) {
method expectString (line 125) | public TestRecord expectString(String field, String... values) {
method expectLong (line 136) | public TestRecord expectLong(String field, Long... values) {
method expectDouble (line 147) | public TestRecord expectDouble(String field, Double... values) {
method isPresent (line 158) | private void isPresent(Map<String, ?> results, String field, Object va...
method noString (line 180) | public TestRecord noString(String field) {
method noLong (line 185) | public TestRecord noLong(String field) {
method noDouble (line 190) | public TestRecord noDouble(String field) {
method isAbsent (line 195) | private void isAbsent(Map<String, ?> results, String field) {
method clear (line 205) | public void clear() {
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/TestUltimateDummyDissector.java
class TestUltimateDummyDissector (line 21) | class TestUltimateDummyDissector {
method verifyUltimateDummyDissector (line 23) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/TestUltimateDummyDissectorFailurelogging.java
class TestUltimateDummyDissectorFailurelogging (line 24) | class TestUltimateDummyDissectorFailurelogging {
method verifyErrorSituation (line 26) | @Test
FILE: parser-core/src/test/java/nl/basjes/parse/core/test/UltimateDummyDissector.java
class UltimateDummyDissector (line 33) | public abstract class UltimateDummyDissector extends SimpleDissector {
method UltimateDummyDissector (line 45) | public UltimateDummyDissector() {
method UltimateDummyDissector (line 49) | public UltimateDummyDissector(String inputType) {
method initializeFromSettingsParameter (line 53) | @Override
FILE: utils/PojoGenerator/src/main/java/nl/basjes/parse/httpdlog/PojoGenerator.java
class PojoGenerator (line 28) | public class PojoGenerator {
class MyRecord (line 32) | static class MyRecord {
method setter (line 33) | public void setter(String name, String value) {
method main (line 38) | public static void main(String[] args) throws NoSuchMethodException, M...
method run (line 51) | public void run() throws NoSuchMethodException, MissingDissectorsExcep...
method castToJavaType (line 70) | private String castToJavaType(Casts casts) {
Condensed preview — 212 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (2,209K chars).
[
{
"path": ".editorconfig",
"chars": 767,
"preview": "#\n# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance wit"
},
{
"path": ".github/FUNDING.yml",
"chars": 62,
"preview": "github: nielsbasjes\ncustom: https://www.paypal.me/nielsbasjes\n"
},
{
"path": ".github/workflows/build.yml",
"chars": 1667,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": ".gitignore",
"chars": 378,
"preview": "*.class\n\n# Maven\ndependency-reduced-pom.xml\n\n# Package Files #\n*.jar\n*.war\n*.ear\n\ntarget\n*.bak\n*.swp\n*~\nbin\n__*\n\n#Eclips"
},
{
"path": ".pmd",
"chars": 29777,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": ".travis.yml__",
"chars": 803,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "CHANGELOG.md",
"chars": 7620,
"preview": "This is intended as an overview of the major changes\n\nv6.0.1-SNAPSHOT\n===\n- ...\n\nv6.0.0\n===\n- Updated to need Java 21!\n-"
},
{
"path": "CNAME",
"chars": 19,
"preview": "logparser.basjes.nl"
},
{
"path": "GeoIP2-TestData/Dockerfile",
"chars": 861,
"preview": "# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apache"
},
{
"path": "GeoIP2-TestData/README.md",
"chars": 497,
"preview": "I copied some of the files from https://github.com/maxmind/MaxMind-DB/\nand modified them to suit my own needs.\n\nThese ba"
},
{
"path": "GeoIP2-TestData/rebuild.sh",
"chars": 1920,
"preview": "#!/bin/bash\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed unde"
},
{
"path": "GeoIP2-TestData/source-data/GeoIP2-City-Test.json",
"chars": 3549,
"preview": "[\n {\n \"::80.100.47.45/118\": {\n \"city\": {\n \"confidence\": 1,\n \"geoname_id\": \"1234\",\n \"names\""
},
{
"path": "GeoIP2-TestData/source-data/GeoIP2-Country-Test.json",
"chars": 2341,
"preview": "[\n {\n \"::80.100.47.45/118\": {\n \"continent\": {\n \"code\": \"EU\",\n \"geoname_id\": 6255148,\n \"nam"
},
{
"path": "GeoIP2-TestData/source-data/GeoIP2-ISP-Test.json",
"chars": 442,
"preview": "[\n {\n \"::80.100.47.45/118\": {\n \"autonomous_system_number\": 4444,\n \"autonomous_system_organization\": \"Basje"
},
{
"path": "GeoIP2-TestData/source-data/GeoLite2-ASN-Test.json",
"chars": 302,
"preview": "[\n {\n \"::80.100.47.45/118\": {\n \"autonomous_system_number\": 4444,\n \"autonomous_system_organization\": \"Basje"
},
{
"path": "GeoIP2-TestData/test-data/write-test-data.pl",
"chars": 21641,
"preview": "#!/usr/bin/env perl\n\nuse strict;\nuse warnings;\nuse autodie;\nuse utf8;\n\nuse Cwd qw( abs_path );\nuse File::Basename qw( di"
},
{
"path": "LICENSE",
"chars": 11360,
"preview": "\n Apache License\n Version 2.0, January 2004\n "
},
{
"path": "README-Hive.md",
"chars": 5103,
"preview": "Hive\n====\n\nThe SerDe (it's really only a Deserializer) can be used present an Apache HTTPD logfile as a table in Hive.\n\n"
},
{
"path": "README-Java.md",
"chars": 6373,
"preview": "Apache HTTPD logparser\n===\nThis is a Logparsing framework intended to make parsing Apache HTTPD logfiles much easier.\n\nT"
},
{
"path": "README-Pig.md",
"chars": 850,
"preview": "Abandoned!\n===\nVersion 5.8 is the last version to support Apache Pig.\nThe last release of Apache Pig was in 2017 and rig"
},
{
"path": "README-geoip.md",
"chars": 2785,
"preview": "Dissect IP using GeoIP2 information\n===\nThis project also contains a dissector that uses the [MaxMind](https://www.maxmi"
},
{
"path": "README.md",
"chars": 12076,
"preview": "Apache HTTPD & NGINX access log parser\n======================================\n[![Github actions Build status](https://im"
},
{
"path": "_config.yml",
"chars": 26,
"preview": "theme: jekyll-theme-cayman"
},
{
"path": "devtools/docker/Dockerfile",
"chars": 3373,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "devtools/docker/bashcolors.sh",
"chars": 2976,
"preview": "#!/bin/bash\n\n#\n# Yet Another UserAgent Analyzer\n# Copyright (C) 2013-2022 Niels Basjes\n#\n# Licensed under the Apache Lic"
},
{
"path": "devtools/docker/build_env_checks.sh",
"chars": 3826,
"preview": "#!/bin/bash\n\n#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed u"
},
{
"path": "devtools/docker/configure-for-user.sh",
"chars": 1254,
"preview": "#!/bin/bash\n\n#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed u"
},
{
"path": "devtools/docker/env.sh",
"chars": 2024,
"preview": "#!/bin/bash\n#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed un"
},
{
"path": "devtools/docker/prompt.sh",
"chars": 2928,
"preview": "#!/bin/bash\n\n#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed u"
},
{
"path": "devtools/logformat.conf",
"chars": 5312,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "devtools/pom.xml",
"chars": 1792,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "devtools/release.sh",
"chars": 6612,
"preview": "#!/bin/bash\n#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2025 Niels Basjes\n#\n# Licensed un"
},
{
"path": "devtools/src/main/resources/checkstyle/checkstyle.xml",
"chars": 7288,
"preview": "<?xml version=\"1.0\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Li"
},
{
"path": "devtools/src/main/resources/checkstyle/suppressions.xml",
"chars": 1201,
"preview": "<?xml version=\"1.0\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Li"
},
{
"path": "docs/CNAME",
"chars": 19,
"preview": "logparser.basjes.nl"
},
{
"path": "docs/README.md",
"chars": 2765,
"preview": "Apache HTTPD & NGINX access log parser\n======================================\n[![Github actions Build status](https://im"
},
{
"path": "examples/apache-beam/.gitignore",
"chars": 4,
"preview": "tmp\n"
},
{
"path": "examples/apache-beam/pom.xml",
"chars": 4483,
"preview": "<?xml version=\"1.0\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Li"
},
{
"path": "examples/apache-beam/src/test/avro/Record.avdl",
"chars": 2405,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/TestCase.java",
"chars": 9456,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/ExpectedClick.java",
"chars": 3977,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvro.java",
"chars": 7977,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/avro/TestParserDoFnAvroInline.java",
"chars": 8034,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/MyRecord.java",
"chars": 7885,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/TestParserDoFnClass.java",
"chars": 2831,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/java/nl/basjes/parse/httpdlog/beam/pojo/TestParserDoFnInline.java",
"chars": 2773,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-beam/src/test/resources/log4j.properties",
"chars": 1324,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "examples/apache-flink/.gitignore",
"chars": 4,
"preview": "tmp\n"
},
{
"path": "examples/apache-flink/pom.xml",
"chars": 5010,
"preview": "<?xml version=\"1.0\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Li"
},
{
"path": "examples/apache-flink/src/test/avro/Record.avdl",
"chars": 2406,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/TestCase.java",
"chars": 9487,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/ExpectedClick.java",
"chars": 3997,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/TestParserMapFunctionAvroClass.java",
"chars": 7591,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/avro/TestParserMapFunctionAvroInline.java",
"chars": 7438,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/MyRecord.java",
"chars": 8022,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/TestParserMapFunctionClass.java",
"chars": 2423,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/java/nl/basjes/parse/httpdlog/flink/pojo/TestParserMapFunctionInline.java",
"chars": 2430,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/apache-flink/src/test/resources/log4j.properties",
"chars": 1324,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "examples/apache-hadoop-mapreduce/.gitignore",
"chars": 20,
"preview": "debug-output\noutput\n"
},
{
"path": "examples/apache-hadoop-mapreduce/pom.xml",
"chars": 3140,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "examples/apache-hadoop-mapreduce/src/main/assembly/job.xml",
"chars": 1672,
"preview": "<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Licensed under the Apach"
},
{
"path": "examples/apache-hadoop-mapreduce/src/main/java/nl/basjes/hadoop/io/input/Wordcount.java",
"chars": 5131,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/demolog/README.md",
"chars": 1290,
"preview": "The file hackers-access.log is a sample of the access logs from my home webserver.\n\nThis file is in the 'combined' LogFo"
},
{
"path": "examples/demolog/hackers-access.log",
"chars": 796435,
"preview": "195.154.46.135 - - [25/Oct/2015:04:11:25 +0100] \"GET /linux/doing-pxe-without-dhcp-control HTTP/1.1\" 200 24323 \"http://h"
},
{
"path": "examples/java-pojo/pom.xml",
"chars": 1804,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "examples/java-pojo/src/main/java/nl/basjes/parse/Main.java",
"chars": 6066,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/java-pojo/src/main/java/nl/basjes/parse/MyRecord.java",
"chars": 2352,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "examples/java-pojo/src/main/resources/log4j.properties",
"chars": 1325,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "examples/pom.xml",
"chars": 2433,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "httpdlog/httpdlog-inputformat/pom.xml",
"chars": 2155,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "httpdlog/httpdlog-inputformat/src/main/assembly/job.xml",
"chars": 1754,
"preview": "<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Licensed under the Apach"
},
{
"path": "httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ApacheHttpdLogfileInputFormat.java",
"chars": 4472,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ApacheHttpdLogfileRecordReader.java",
"chars": 11739,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-inputformat/src/main/java/nl/basjes/hadoop/input/ParsedRecord.java",
"chars": 6879,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestApacheHttpdLogfileInputFormat.java",
"chars": 6844,
"preview": "package nl.basjes.hadoop.input;\n/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels"
},
{
"path": "httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestGetAllFields.java",
"chars": 3629,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-inputformat/src/test/java/nl/basjes/hadoop/input/TestParsedRecord.java",
"chars": 5951,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-inputformat/src/test/resources/access.log",
"chars": 304,
"preview": "127.0.0.1 - - [01/Jan/2017:13:01:21 +0100] \"GET / HTTP/1.1\" 200 3525 \"01/01/17 2017-01-01 13:01 13:01:21 01:01:21 PM Sun"
},
{
"path": "httpdlog/httpdlog-inputformat/src/test/resources/log4j.properties",
"chars": 1325,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "httpdlog/httpdlog-parser/pom.xml",
"chars": 9905,
"preview": "<?xml version=\"1.0\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Li"
},
{
"path": "httpdlog/httpdlog-parser/src/main/antlr4/nl/basjes/parse/strftime/StrfTime.g4",
"chars": 7170,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/assembly/job.xml",
"chars": 1672,
"preview": "<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Licensed under the Apach"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdLogFormatDissector.java",
"chars": 32866,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/HttpdLogFormatDissector.java",
"chars": 9924,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/HttpdLoglineParser.java",
"chars": 4811,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/NginxHttpdLogFormatDissector.java",
"chars": 8021,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/Utils.java",
"chars": 11896,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpFirstLineDissector.java",
"chars": 5980,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpFirstLineProtocolDissector.java",
"chars": 3617,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/HttpUriDissector.java",
"chars": 10482,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ModUniqueIdDissector.java",
"chars": 9304,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/QueryStringFieldDissector.java",
"chars": 3837,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/RequestCookieListDissector.java",
"chars": 4092,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ResponseSetCookieDissector.java",
"chars": 5225,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ResponseSetCookieListDissector.java",
"chars": 4020,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/ScreenResolutionDissector.java",
"chars": 3024,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeStampDissector.java",
"chars": 5028,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/StrfTimeToDateTimeFormatter.java",
"chars": 16978,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/TimeStampDissector.java",
"chars": 23961,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/AbstractGeoIPDissector.java",
"chars": 3685,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPASNDissector.java",
"chars": 3787,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPCityDissector.java",
"chars": 9242,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPCountryDissector.java",
"chars": 6467,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/geoip/GeoIPISPDissector.java",
"chars": 3314,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/CoreLogModule.java",
"chars": 20577,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/GeoIPModule.java",
"chars": 5273,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/KubernetesIngressModule.java",
"chars": 2686,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/NginxModule.java",
"chars": 1078,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/SslModule.java",
"chars": 9827,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/UpstreamListDissector.java",
"chars": 6375,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/UpstreamModule.java",
"chars": 10818,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/nginxmodules/VariousModule.java",
"chars": 13101,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/NamedTokenParser.java",
"chars": 3109,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/ParameterizedTokenParser.java",
"chars": 4635,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/Token.java",
"chars": 3472,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenFormatDissector.java",
"chars": 14290,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenOutputField.java",
"chars": 2494,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenParser.java",
"chars": 8896,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/tokenformat/TokenSorterByStartPos.java",
"chars": 1301,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertCLFIntoNumber.java",
"chars": 1491,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertMillisecondsIntoMicroseconds.java",
"chars": 1344,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertNumberIntoCLF.java",
"chars": 1282,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/ConvertSecondsWithMillisStringDissector.java",
"chars": 1579,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/java/nl/basjes/parse/httpdlog/dissectors/translate/TypeConvertBaseDissector.java",
"chars": 2056,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/main/resources/version/Version.java.template",
"chars": 1297,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ApacheHttpdAllFieldsTest.java",
"chars": 26605,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ApacheHttpdLogParserTest.java",
"chars": 32744,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/BasicOverallTest.java",
"chars": 26941,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/ClientHintsTest.java",
"chars": 3993,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/CookiesTest.java",
"chars": 10887,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/EdgeCasesTest.java",
"chars": 4931,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/JettyLogFormatParserTest.java",
"chars": 4398,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/JsonLogFormatTest.java",
"chars": 9471,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/MultiLineHttpdLogParserTest.java",
"chars": 5192,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatJsonTest.java",
"chars": 2306,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/NginxLogFormatTest.java",
"chars": 45023,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/UtilsTest.java",
"chars": 9518,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestCookieDissector.java",
"chars": 6696,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestGeoIPDissectors.java",
"chars": 14979,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestHttpFirstLineDissector.java",
"chars": 4330,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestHttpUriDissector.java",
"chars": 9937,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestModUniqueIdDissector.java",
"chars": 3626,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestQueryStringDissector.java",
"chars": 1608,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/dissectors/TestTimeStampDissector.java",
"chars": 42172,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/nginxmodules/NginxAllFieldsTest.java",
"chars": 20929,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/nginxmodules/NginxUpstreamTest.java",
"chars": 23969,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/java/nl/basjes/parse/httpdlog/translate/TestTranslators.java",
"chars": 2369,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-parser/src/test/resources/log4j.properties",
"chars": 1325,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "httpdlog/httpdlog-serde/pom.xml",
"chars": 5916,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "httpdlog/httpdlog-serde/src/main/assembly/udf.xml",
"chars": 1823,
"preview": "<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Licensed under the Apach"
},
{
"path": "httpdlog/httpdlog-serde/src/main/java/nl/basjes/parse/httpdlog/ApacheHttpdlogDeserializer.java",
"chars": 13853,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-serde/src/test/java/nl/basjes/parse/httpdlog/TestAllDissectorTypes.java",
"chars": 6009,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-serde/src/test/java/nl/basjes/parse/httpdlog/TestApacheHttpdlogDeserializer.java",
"chars": 5336,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "httpdlog/httpdlog-serde/src/test/resources/log4j.properties",
"chars": 1325,
"preview": "#\n# Apache HTTPD & NGINX Access log parsing made easy\n# Copyright (C) 2011-2023 Niels Basjes\n#\n# Licensed under the Apac"
},
{
"path": "httpdlog/pom.xml",
"chars": 1548,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "parser-core/pom.xml",
"chars": 1972,
"preview": "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 "
},
{
"path": "parser-core/src/main/assembly/job.xml",
"chars": 1672,
"preview": "<!--\n Apache HTTPD & NGINX Access log parsing made easy\n Copyright (C) 2011-2023 Niels Basjes\n\n Licensed under the Apach"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Casts.java",
"chars": 1351,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Dissector.java",
"chars": 7925,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Field.java",
"chars": 1150,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Parsable.java",
"chars": 9277,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/ParsedField.java",
"chars": 1655,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Parser.java",
"chars": 43249,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/SimpleDissector.java",
"chars": 3121,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/Value.java",
"chars": 2840,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/exceptions/DissectionFailure.java",
"chars": 982,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/exceptions/FatalErrorDuringCallOfSetterMethod.java",
"chars": 1118,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/exceptions/InvalidDissectorException.java",
"chars": 1131,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/exceptions/InvalidFieldMethodSignature.java",
"chars": 1115,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/main/java/nl/basjes/parse/core/exceptions/MissingDissectorsException.java",
"chars": 899,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserCastsTest.java",
"chars": 9752,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserDissectionOutputTypesTest.java",
"chars": 9337,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserDuplicateOutputTest.java",
"chars": 2354,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserExceptionsTest.java",
"chars": 12651,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserInfiniteLoopTest.java",
"chars": 2885,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserNormalTest.java",
"chars": 11132,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserNormalTestRecord.java",
"chars": 3740,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserResetTest.java",
"chars": 4308,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserTypeColissionTest.java",
"chars": 5825,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/ParserTypeRemappingEdgeCase.java",
"chars": 3626,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/TestBadAPIUsage.java",
"chars": 7713,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSetters.java",
"chars": 22982,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersAlwaysCombined.java",
"chars": 3665,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersAlwaysSeparate.java",
"chars": 4776,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersNotEmpty.java",
"chars": 5146,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/annotation/TestFieldSettersNotNull.java",
"chars": 5125,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/convert/ValueConvertTest.java",
"chars": 4648,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/reference/BarDissector.java",
"chars": 2417,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/reference/FooDissector.java",
"chars": 2417,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/reference/FooSpecialDissector.java",
"chars": 995,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/reference/ReferenceTest.java",
"chars": 7820,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/reference/ReferenceTestDouble.java",
"chars": 8054,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/DissectorTester.java",
"chars": 31577,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/EmptyValuesDissector.java",
"chars": 1732,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/MyDissectorTester.java",
"chars": 1450,
"preview": "/*\n * Apache HTTPD logparsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the Apache License,"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/NormalValuesDissector.java",
"chars": 1750,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/NullValuesDissector.java",
"chars": 1793,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/TestRecord.java",
"chars": 7010,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/TestUltimateDummyDissector.java",
"chars": 1821,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
},
{
"path": "parser-core/src/test/java/nl/basjes/parse/core/test/TestUltimateDummyDissectorFailurelogging.java",
"chars": 5706,
"preview": "/*\n * Apache HTTPD & NGINX Access log parsing made easy\n * Copyright (C) 2011-2023 Niels Basjes\n *\n * Licensed under the"
}
]
// ... and 12 more files (download for full content)
About this extraction
This page contains the full source code of the nielsbasjes/logparser GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 212 files (2.0 MB), approximately 540.2k tokens, and a symbol index with 1511 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.