Merge branch 'master' into bug/helpdesk-horizontal-multi-column

data-service/README.md

@@ -0,0 +1,2 @@
+# pwm-data-service
+Cloud service for PWM

data-service/checkstyle.xml

@@ -0,0 +1,325 @@
+<?xml version="1.0"?>
+<!--
+  ~ Password Management Servlets (PWM)
+  ~ http://www.pwm-project.org
+  ~
+  ~ Copyright (c) 2006-2009 Novell, Inc.
+  ~ Copyright (c) 2009-2016 The PWM Project
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or
+  ~ (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+  -->
+
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.2//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_2.dtd">
+
+<!--
+  PWM Checkstyle definition
+-->
+
+<module name="Checker">
+
+    <!-- Checks that each Java package has a Javadoc file used for commenting. -->
+    <!-- See http://checkstyle.sf.net/config_javadoc.html#JavadocPackage       -->
+    <!--module name="JavadocPackage">
+      <property name="allowLegacy" value="true"/>
+    </module-->
+
+    <module name="FileLength"/>
+
+    <!-- Checks for Headers                              -->
+    <!-- See http://checkstyle.sf.net/config_header.html -->
+    <!--
+    <module name="RegexpHeader">
+        <property name="fileExtensions" value="java"/>
+        <property name="headerFile" value="${checkstyle.header.file}"/>
+    </module>
+    -->
+
+    <module name="FileTabCharacter">
+        <property name="eachLine" value="true"/>
+    </module>
+    <module name="NewlineAtEndOfFile"/>
+
+    <module name="TreeWalker">
+
+        <property name="cacheFile" value="target/checkstyle.cache"/>
+
+        <!-- required for SuppressWarningsFilter (and other Suppress* rules not used here) -->
+        <!-- see http://checkstyle.sourceforge.net/config_annotation.html#SuppressWarningsHolder -->
+        <module name="SuppressWarningsHolder"/>
+
+        <module name="OuterTypeFilename"/>
+        <module name="IllegalTokenText">
+            <property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
+            <property name="format" value="\\u00(08|09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
+            <property name="message" value="Avoid using corresponding octal or Unicode escape."/>
+        </module>
+        <module name="AvoidEscapedUnicodeCharacters">
+            <property name="allowEscapesForControlCharacters" value="true"/>
+            <property name="allowByTailComment" value="true"/>
+            <property name="allowNonPrintableEscapes" value="true"/>
+        </module>
+
+        <!--
+        <module name="LineLength">
+            <property name="max" value="200" />
+            <property name="ignorePattern" value="@version|@see|@todo|TODO"/>
+        </module>
+        -->
+        <!-- required for SuppressionCommentFilter -->
+        <!-- see http://checkstyle.sourceforge.net/config.html#SuppressionCommentFilter -->
+        <!--
+        <module name="FileContentsHolder"/>
+
+
+        -->
+
+        <module name="EmptyBlock">
+            <property name="option" value="TEXT"/>
+            <property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
+        </module>
+        <!--
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="maxLineLength" value="100"/>
+        </module>
+        -->
+
+        <module name="RightCurly"/>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+            <property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
+        </module>
+
+        <!--
+        <module name="MemberName" />
+        -->
+
+        <!-- Checks for Javadoc comments.                     -->
+        <!-- See http://checkstyle.sf.net/config_javadoc.html -->
+        <!--
+        <module name="JavadocMethod">
+            <property name="severity" value="warning"/>
+            <property name="scope" value="protected"/>
+        </module>
+        <module name="JavadocType">
+            <property name="scope" value="protected"/>
+            <property name="allowUnknownTags" value="true" />
+        </module>
+        <module name="JavadocVariable">
+            <property name="severity" value="info"/>
+            <property name="scope" value="protected"/>
+        </module>
+        -->
+
+        <module name="AnnotationLocation">
+            <property name="tokens" value="VARIABLE_DEF"/>
+            <property name="allowSamelineMultipleAnnotations" value="true"/>
+        </module>
+
+        <!-- Checks for Naming Conventions.                  -->
+        <!-- See http://checkstyle.sf.net/config_naming.html -->
+        <!--
+        <module name="MemberName">
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+        </module>
+        <module name="TypeName">
+        -->
+        <module name="ConstantName"/>
+        <module name="PackageName">
+            <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
+        </module>
+        <module name="LocalVariableName">
+            <property name="tokens" value="VARIABLE_DEF"/>
+            <property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
+            <property name="allowOneCharVarInForLoop" value="true"/>
+        </module>
+        <!--
+        <module name="ClassTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+        </module>
+        -->
+        <module name="MethodTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+        </module>
+        <module name="InterfaceTypeParameterName">
+            <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
+        </module>
+        <!--
+        <module name="LocalFinalVariableName"/>
+        <module name="LocalVariableName"/>
+        <module name="MethodName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="TypeName"/>
+        -->
+
+        <!-- Checks for imports                              -->
+        <!-- See http://checkstyle.sf.net/config_import.html -->
+        <module name="AvoidStarImport"/>
+        <module name="AvoidStaticImport"/>
+        <module name="IllegalImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+
+
+        <!-- Checks for Size Violations.                    -->
+        <!-- See http://checkstyle.sf.net/config_sizes.html -->
+        <!--
+        <module name="MethodLength"/>
+        <module name="ParameterNumber"/>
+        -->
+
+
+        <!-- Checks for whitespace                               -->
+        <!-- See http://checkstyle.sf.net/config_whitespace.html -->
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="EmptyForInitializerPad"/>
+        <module name="NeedBraces"/>
+        <!--
+        -->
+        <!-- module name="NoWhitespaceAfter"/ -->
+        <!-- module name="NoWhitespaceBefore"/ -->
+        <!--
+        <module name="OperatorWrap"/>
+        <module name="ParenPad">
+            <property name="option" value="space" />
+        </module>
+        <module name="WhitespaceAfter"/>
+        <module name="WhitespaceAround"/>
+        -->
+        <!-- module name="MethodParamPad"/ -->
+        <module name="GenericWhitespace"/>
+        <module name="EmptyLineSeparator">
+            <property name="allowNoEmptyLineBetweenFields" value="true"/>
+        </module>
+
+
+
+        <!-- Modifier Checks                                    -->
+        <!-- See http://checkstyle.sf.net/config_modifiers.html -->
+        <module name="ModifierOrder"/>
+        <module name="RedundantModifier"/>
+        <!--
+        -->
+
+
+        <!-- Checks for blocks. You know, those {}'s         -->
+        <!-- See http://checkstyle.sf.net/config_blocks.html -->
+        <!--
+        <module name="AvoidNestedBlocks"/>
+        -->
+
+
+        <!-- Checks for common coding problems               -->
+        <!-- See http://checkstyle.sf.net/config_coding.html -->
+        <!-- module name="AvoidInlineConditionals"/ -->
+        <!--
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="HiddenField">
+            <property name="severity" value="warning"/>
+            <property name="ignoreSetter" value="true"/>
+            <property name="ignoreConstructorParameter" value="true"/>
+        </module>
+        <module name="IllegalInstantiation"/>
+        <module name="InnerAssignment"/>
+        -->
+        <!--
+        <module name="MagicNumber">
+            <property name="ignoreNumbers" value="-4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 31, 32, 37, 64, 100, 128, 256, 512, 1000, 1024"/>
+        </module>
+        -->
+
+        <!-- Checks for class design                         -->
+        <!-- See http://checkstyle.sf.net/config_design.html -->
+        <!-- module name="DesignForExtension"/ -->
+        <!-- module name="FinalClass"/ -->
+        <!-- module name="HideUtilityClassConstructor"/ -->
+        <!--
+        <module name="InterfaceIsType"/>
+        <module name="VisibilityModifier">
+            <property name="protectedAllowed" value="true"/>
+            <property name="packageAllowed" value="true"/>
+        </module>
+        -->
+
+
+        <!-- future enabled checks -->
+        <!--
+        <module name="TrailingComment"/>
+        <module name="NPathComplexity"/>
+        <module name="EnumTrailingCommaCheck"/> //doesnt yet exist as of checkstyle 2.17
+        <module name="MultipleStringLiterals"/>
+        <module name="InnerAssignment"/>
+        <module name="MagicNumber">
+            <property name="ignoreNumbers" value="-4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 31, 32, 37, 64, 100, 128, 256, 512, 1000, 1024"/>
+        </module>
+        <module name="SimplifyBooleanExpression"/>
+        -->
+
+        <!-- coding -->
+        <module name="FallThrough"/>
+        <module name="EqualsHashCode"/>
+        <module name="ArrayTrailingCommaCheck"/>
+        <module name="FinalLocalVariable"/>
+        <module name="MissingSwitchDefault"/>
+        <module name="ModifiedControlVariable"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="OneStatementPerLine"/>
+        <module name="FinalParameters"/>
+        <module name="ParameterAssignment"/>
+        <module name="SimplifyBooleanReturn"/>
+        <module name="StringLiteralEquality"/>
+        <module name="CovariantEquals"/>
+        <module name="DefaultComesLast"/>
+        <module name="EmptyStatement"/>
+        <module name="EqualsHashCode"/>
+        <module name="EqualsAvoidNull"/>
+
+        <module name="MutableException"/>
+        <module name="TodoComment"/>
+        <module name="NoLineWrap"/>
+        <module name="OneTopLevelClass"/>
+        <module name="NoFinalizer"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="UpperEll"/>
+        <module name="PackageDeclaration"/>
+        <module name="NoClone"/>
+    </module>
+
+    <!-- Support @SuppressWarnings (added in Checkstyle 5.7) -->
+    <!-- see http://checkstyle.sourceforge.net/config.html#SuppressWarningsFilter -->
+    <module name="SuppressWarningsFilter"/>
+
+    <!-- Checks properties file for a duplicated properties. -->
+    <!-- See http://checkstyle.sourceforge.net/config_misc.html#UniqueProperties -->
+    <module name="UniqueProperties"/>
+
+    <!-- Support CHECKSTYLE_OFF: regexp and CHECKSTYLE_ON: regexp comments to disable/enable some checks -->
+    <!-- see http://checkstyle.sourceforge.net/config.html#SuppressionCommentFilter -->
+    <!--
+    <module name="SuppressionCommentFilter">
+        <property name="offCommentFormat" value="CHECKSTYLE_OFF\: (.+)"/>
+        <property name="onCommentFormat" value="CHECKSTYLE_ON\: (.+)"/>
+        <property name="checkFormat" value="$1"/>
+    </module>
+    -->
+
+</module>

data-service/pom.xml

@@ -0,0 +1,335 @@
+<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <parent>
+        <groupId>org.pwm-project</groupId>
+        <artifactId>pwm-parent</artifactId>
+        <version>1.8.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>pwm-data-service</artifactId>
+
+    <packaging>war</packaging>
+
+    <name>PWM Password Self Service: Data Service</name>
+
+    <licenses>
+        <license>
+            <name>The GNU General Public License (GPL) Version 2</name>
+            <url>http://www.gnu.org/licenses/gpl-2.0.html</url>
+            <distribution>repo</distribution>
+        </license>
+    </licenses>
+
+    <organization>
+        <name>PWM Project</name>
+        <url>http://www.pwm-project.org</url>
+    </organization>
+
+    <properties>
+        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.target>1.8</maven.compiler.target>
+        <skipTests>false</skipTests>
+        <timestamp.iso>${maven.build.timestamp}</timestamp.iso>
+        <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ss'Z'</maven.build.timestamp.format>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <build.number>0</build.number>  <!-- default in case not set on command line -->
+        <build.revision>0</build.revision>  <!-- default in case not set on command line -->
+    </properties>
+
+    <profiles>
+        <profile>
+            <id>skip-all</id>
+            <properties>
+                <maven.javadoc.skip>true</maven.javadoc.skip>
+                <source.skip>true</source.skip>
+                <skipTests>true</skipTests>
+                <checkstyle.skip>true</checkstyle.skip>
+                <skip.npm>true</skip.npm>
+            </properties>
+        </profile>
+        <profile>
+            <id>skip-tests</id>
+            <properties>
+                <skipTests>true</skipTests>
+            </properties>
+        </profile>
+        <profile>
+            <id>skip-checkstyle</id>
+            <properties>
+                <checkstyle.skip>true</checkstyle.skip>
+            </properties>
+        </profile>
+    </profiles>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+                <version>3.0.0-M2</version>
+                <executions>
+                    <execution>
+                        <id>enforce-maven</id>
+                        <goals>
+                            <goal>enforce</goal>
+                        </goals>
+                        <configuration>
+                            <rules>
+                                <requireMavenVersion>
+                                    <version>3.3</version>
+                                </requireMavenVersion>
+                            </rules>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <!-- This plugin will set properties values using dependency information -->
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>properties</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.3</version>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.19.1</version>
+                <configuration>
+                    <excludes>
+                        <exclude>password.pwm.manual.*</exclude>
+                    </excludes>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-source-plugin</artifactId>
+                <version>2.4</version>
+                <executions>
+                    <execution>
+                        <id>attach-sources</id>
+                        <goals>
+                            <goal>jar-no-fork</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <version>2.10.3</version>
+                <executions>
+                    <execution>
+                        <id>attach-javadocs</id>
+                        <goals>
+                            <goal>jar</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-war-plugin</artifactId>
+                <version>2.6</version>
+                <configuration>
+                    <archiveClasses>true</archiveClasses>
+                    <packagingExcludes>WEB-INF/classes</packagingExcludes>
+                    <archive>
+                        <manifestEntries>
+                            <Implementation-Title>${project.name}</Implementation-Title>
+                            <Implementation-Version>${project.version}</Implementation-Version>
+                            <Implementation-Vendor>${project.organization.name}</Implementation-Vendor>
+                            <Implementation-URL>${project.organization.url}</Implementation-URL>
+                            <Implementation-Build>${build.number}</Implementation-Build>
+                            <Implementation-Revision>${build.revision}</Implementation-Revision>
+                            <Implementation-Version-Display>v${project.version} b${build.number} r${build.revision}</Implementation-Version-Display>
+                        </manifestEntries>
+                    </archive>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-checkstyle-plugin</artifactId>
+                <version>3.0.0</version>
+                <dependencies>
+                    <dependency>
+                        <groupId>com.puppycrawl.tools</groupId>
+                        <artifactId>checkstyle</artifactId>
+                        <version>8.10.1</version>
+                    </dependency>
+                </dependencies>
+                <executions>
+                    <execution>
+                        <id>validate</id>
+                        <phase>validate</phase>
+                        <configuration>
+                            <encoding>UTF-8</encoding>
+                            <consoleOutput>true</consoleOutput>
+                            <includeTestResources>false</includeTestResources>
+                            <failsOnError>true</failsOnError>
+                            <configLocation>checkstyle.xml</configLocation>
+                        </configuration>
+                        <goals>
+                            <goal>check</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-resources-plugin</artifactId>
+                <version>2.7</version>
+                <executions>
+                    <execution>
+                        <id>copy-resources</id>
+                        <phase>validate</phase>
+                        <goals>
+                            <goal>copy-resources</goal>
+                        </goals>
+                        <configuration>
+                            <outputDirectory>${project.build.outputDirectory}/src</outputDirectory>
+                            <resources>
+                                <resource><directory>src/main/java</directory></resource>
+                                <resource><directory>src/main/resources</directory></resource>
+                            </resources>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <artifactId>maven-clean-plugin</artifactId>
+                <version>3.0.0</version>
+            </plugin>
+        </plugins>
+    </build>
+
+    <reporting>
+    </reporting>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>pwm</artifactId>
+            <version>${project.version}</version>
+            <type>jar</type>
+        </dependency>
+
+        <!-- dev tool -->
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>1.18.0</version>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- container dependencies -->
+        <dependency>
+            <groupId>javax.servlet</groupId>
+            <artifactId>javax.servlet-api</artifactId>
+            <version>4.0.1</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>javax.servlet.jsp</groupId>
+            <artifactId>jsp-api</artifactId>
+            <version>2.2.1-b03</version>
+            <scope>provided</scope>
+        </dependency>
+        <!-- / container dependencies -->
+
+        <dependency>
+            <groupId>commons-net</groupId>
+            <artifactId>commons-net</artifactId>
+            <version>3.6</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-csv</artifactId>
+            <version>1.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.7</version>
+        </dependency>
+        <dependency>
+            <groupId>com.sun.mail</groupId>
+            <artifactId>javax.mail</artifactId>
+            <version>1.6.1</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+            <version>4.5.5</version>
+        </dependency>
+        <dependency>
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>1.2.17</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.axis</groupId>
+            <artifactId>axis</artifactId>
+            <version>1.4</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jdom</groupId>
+            <artifactId>jdom2</artifactId>
+            <version>2.0.6</version>
+        </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+            <version>2.8.5</version>
+        </dependency>
+        <dependency>
+            <groupId>org.jetbrains.xodus</groupId>
+            <artifactId>xodus-environment</artifactId>
+            <version>1.2.3</version>
+        </dependency>
+        <dependency>
+            <groupId>org.webjars</groupId>
+            <artifactId>webjars-locator-core</artifactId>
+            <version>0.35</version>
+        </dependency>
+
+
+
+    </dependencies>
+
+    <repositories>
+        <repository>
+            <id>central</id>
+            <url>https://repo1.maven.org/maven2</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>central</id>
+            <url>https://repo1.maven.org/maven2</url>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </pluginRepository>
+    </pluginRepositories>
+</project>

data-service/src/main/java/password/pwm/receiver/ContextManager.java

@@ -0,0 +1,55 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import javax.servlet.annotation.WebListener;
+
+@WebListener
+public class ContextManager implements ServletContextListener {
+    private static final String CONTEXT_ATTR = "contextManager";
+    private PwmReceiverApp app;
+
+    @Override
+    public void contextInitialized(final ServletContextEvent sce) {
+        app = new PwmReceiverApp();
+        sce.getServletContext().setAttribute(CONTEXT_ATTR, this);
+    }
+
+    @Override
+    public void contextDestroyed(final ServletContextEvent sce) {
+        app.close();
+        app = null;
+    }
+
+    public PwmReceiverApp getApp() {
+        return app;
+    }
+
+    public static ContextManager getContextManager(final ServletContext serverContext) {
+        return (ContextManager)serverContext.getAttribute(CONTEXT_ATTR);
+    }
+}

data-service/src/main/java/password/pwm/receiver/CsvDownloadServlet.java

@@ -0,0 +1,35 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import javax.servlet.annotation.WebServlet;
+
+@WebServlet(
+        name="TelemetryViewer",
+        urlPatterns={
+                "/csv",
+        }
+)
+public class CsvDownloadServlet {
+}

data-service/src/main/java/password/pwm/receiver/FtpDataIngestor.java

@@ -0,0 +1,162 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import org.apache.commons.net.ftp.FTPClient;
+import org.apache.commons.net.ftp.FTPFile;
+import org.apache.commons.net.ftp.FTPSClient;
+import password.pwm.PwmConstants;
+import password.pwm.bean.TelemetryPublishBean;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+class FtpDataIngestor {
+
+    private static final PwmReceiverLogger LOGGER = PwmReceiverLogger.forClass(FtpDataIngestor.class);
+
+    private final Settings settings;
+    private final PwmReceiverApp app;
+
+    FtpDataIngestor(final PwmReceiverApp app, final Settings telemetrySettings) {
+        this.app = app;
+        this.settings = telemetrySettings;
+    }
+
+    void readData(final Storage storage) {
+        app.getStatus().setLastFtpStatus("beginning ftp ingestion");
+        LOGGER.debug( "beginning ftp ingestion" );
+        app.getStatus().setLastFtpIngest(Instant.now());
+        try {
+            final FTPClient ftpClient = getFtpClient();
+            final List<String> files = getFiles(ftpClient);
+            LOGGER.debug( "beginning ftp ingestion, listed " + files.size() + " files from server" );
+            for (final String fileName : files) {
+                if (fileName != null && fileName.endsWith(".zip")) {
+                    app.getStatus().setLastFtpIngest(Instant.now());
+                    app.getStatus().setLastFtpStatus("reading file " + fileName);
+                    LOGGER.debug( "read file " + fileName );
+                    try {
+                        readFile( ftpClient, fileName, storage );
+                    } catch (Exception e) {
+                        app.getStatus().setLastFtpIngest(Instant.now());
+                        final String msg = "error while reading ftp file '" + fileName + "': " + e.getMessage();
+                        app.getStatus().setLastFtpStatus(msg);
+                        LOGGER.error( msg );
+                    }
+                } else {
+                    LOGGER.info("skipping ftp file " + fileName);
+                }
+            }
+            ftpClient.disconnect();
+            LOGGER.info("completed ftp ingestion");
+            app.getStatus().setLastFtpStatus("completed successfully");
+            app.getStatus().setLastFtpIngest(Instant.now());
+            app.getStatus().setLastFtpFilesRead( files.size() );
+        } catch (Exception e) {
+            app.getStatus().setLastFtpIngest(Instant.now());
+            app.getStatus().setLastFtpStatus("error during ftp scan: " + e.getMessage());
+        }
+    }
+
+    private void readFile(final FTPClient ftpClient, final String fileName, final Storage storage) throws Exception {
+        final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+        ftpClient.retrieveFile(fileName, byteArrayOutputStream);
+        final ByteArrayInputStream inputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
+        readZippedByteStream(inputStream, fileName, storage);
+    }
+
+    private void readZippedByteStream(final InputStream inputStream, final String fileName, final Storage storage) throws Exception {
+        try {
+            final ZipInputStream zipInputStream = new ZipInputStream(inputStream);
+            final ZipEntry zipEntry = zipInputStream.getNextEntry();
+            final String zipEntryName = zipEntry.getName();
+            if (zipEntryName != null && zipEntryName.endsWith(".json")) {
+                LOGGER.info("reading ftp file " + fileName + ":" + zipEntryName);
+                final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+                final byte[] buffer = new byte[1024];
+                int len;
+                while ((len = zipInputStream.read(buffer)) > 0) {
+                    byteArrayOutputStream.write(buffer, 0, len);
+                }
+                final String resultsStr = byteArrayOutputStream.toString(PwmConstants.DEFAULT_CHARSET.name());
+                final TelemetryPublishBean bean = JsonUtil.deserialize(resultsStr, TelemetryPublishBean.class);
+                storage.store(bean);
+            }
+        } catch (Exception e) {
+            final String msg = "error reading ftp file '" + fileName + "', error: " + e.getMessage();
+            LOGGER.info(msg);
+            throw new Exception(e);
+        }
+    }
+
+    private List<String> getFiles(final FTPClient ftpClient) throws IOException {
+        final String pathname = settings.getSetting( Settings.Setting.ftpReadPath );
+        final FTPFile[] files = ftpClient.listFiles(pathname);
+        final List<String> returnFiles = new ArrayList<>();
+        for (final FTPFile ftpFile : files) {
+            final String name = ftpFile.getName();
+            final String fullPath = pathname + "/" + name;
+            returnFiles.add(fullPath);
+        }
+
+        return Collections.unmodifiableList(returnFiles);
+    }
+
+    private FTPClient getFtpClient() throws IOException {
+        final FTPClient ftpClient;
+        final Settings.FtpMode ftpMode = Settings.FtpMode.valueOf( settings.getSetting( Settings.Setting.ftpMode ) );
+        switch ( ftpMode ) {
+            case ftp:
+                ftpClient = new FTPClient();
+                break;
+
+            case ftps:
+                ftpClient = new FTPSClient();
+                break;
+
+            default:
+                throw new IllegalArgumentException("unexpected ftp mode");
+        }
+
+        ftpClient.connect( settings.getSetting( Settings.Setting.ftpSite ));
+        LOGGER.info("ftp connect complete");
+        if (!StringUtil.isEmpty(settings.getSetting(Settings.Setting.ftpUser)) && !StringUtil.isEmpty(settings.getSetting( Settings.Setting.ftpPassword ))) {
+            final boolean loggedInSuccess = ftpClient.login( settings.getSetting(Settings.Setting.ftpUser), settings.getSetting( Settings.Setting.ftpPassword ));
+            LOGGER.info("ftp login complete, success=" + loggedInSuccess);
+        }
+        ftpClient.enterLocalPassiveMode();
+        return ftpClient;
+    }
+}

data-service/src/main/java/password/pwm/receiver/Logger.java

@@ -0,0 +1,41 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+public class Logger {
+
+    private final String name;
+
+    private Logger(final String name) {
+        this.name = name;
+    }
+
+    public static Logger createLogger(final String name) {
+        return new Logger(name);
+    }
+
+    public void info(final CharSequence input) {
+        System.out.println(input);
+    }
+}

data-service/src/main/java/password/pwm/receiver/PwmReceiverApp.java

@@ -0,0 +1,96 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import password.pwm.util.java.JavaHelper;
+import password.pwm.util.java.StringUtil;
+
+import java.io.IOException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class PwmReceiverApp {
+    private static final PwmReceiverLogger LOGGER = PwmReceiverLogger.forClass( PwmReceiverApp.class );
+    private static final String ENV_NAME = "DATA_SERVICE_PROPS";
+
+    private Storage storage;
+    private ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
+    private Settings settings;
+    private Status status = new Status();
+
+    public PwmReceiverApp() {
+        final String propsFile = System.getenv(ENV_NAME);
+        if (StringUtil.isEmpty(propsFile)) {
+            final String errorMsg = "Missing environment variable '" + ENV_NAME + "', can't load configuration";
+            status.setErrorState( errorMsg );
+            LOGGER.error( errorMsg );
+            return;
+        }
+
+        try {
+            settings = Settings.readFromFile(propsFile);
+        } catch (IOException e) {
+            final String errorMsg = "can't read configuration: " + JavaHelper.readHostileExceptionMessage(e);
+            status.setErrorState( errorMsg );
+            LOGGER.error( errorMsg, e );
+            return;
+        }
+
+        try {
+            storage = new Storage(settings);
+        } catch (Exception e) {
+            final String errorMsg = "can't start storage system: " + JavaHelper.readHostileExceptionMessage(e);
+            status.setErrorState( errorMsg );
+            LOGGER.error( errorMsg, e );
+            return;
+        }
+
+        if (settings.getSetting( Settings.Setting.ftpSite ) != null && !settings.getSetting( Settings.Setting.ftpSite ).isEmpty()) {
+            final Runnable ftpThread = () -> {
+                final FtpDataIngestor ftpDataIngestor = new FtpDataIngestor(this, settings);
+                ftpDataIngestor.readData(storage);
+            };
+            scheduledExecutorService.scheduleAtFixedRate(ftpThread, 0, 1, TimeUnit.HOURS);
+        }
+    }
+
+    public Settings getSettings() {
+        return settings;
+    }
+
+    public Storage getStorage() {
+        return storage;
+    }
+
+    void close() {
+        storage.close();
+        scheduledExecutorService.shutdown();
+    }
+
+    public Status getStatus() {
+        return status;
+    }
+
+}

data-service/src/main/java/password/pwm/receiver/PwmReceiverLogger.java

@@ -0,0 +1,40 @@
+package password.pwm.receiver;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class PwmReceiverLogger
+{
+    private final Class clazz;
+
+    private PwmReceiverLogger( final Class clazz )
+    {
+        this.clazz = clazz;
+    }
+
+    public static PwmReceiverLogger forClass( final Class clazz )
+    {
+        return new PwmReceiverLogger( clazz );
+    }
+
+    public void debug(final CharSequence logMsg) {
+        log( Level.FINE, logMsg, null );
+    }
+
+    public void info(final CharSequence logMsg) {
+        log( Level.INFO, logMsg, null );
+    }
+
+    public void error(final CharSequence logMsg ) {
+        log( Level.SEVERE, logMsg, null );
+    }
+
+    public void error(final CharSequence logMsg, final Throwable throwable ) {
+        log( Level.SEVERE, logMsg, throwable );
+    }
+
+    private void log( final Level level, final CharSequence logMsg, final Throwable throwable ) {
+        final Logger logger = Logger.getLogger(clazz.getName());
+        logger.log( level, logMsg.toString(), throwable );
+    }
+}

data-service/src/main/java/password/pwm/receiver/Settings.java

@@ -0,0 +1,95 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class Settings {
+    enum Setting {
+        ftpMode(FtpMode.ftp.name()),
+        ftpSite(null),
+        ftpUser(null),
+        ftpPassword(null),
+        ftpReadPath(null),
+        storagePath(null),
+        maxInstanceSeconds(Long.toString( new TimeDuration(14, TimeUnit.DAYS).getTotalSeconds() ) ),
+
+        ;
+
+        private final String defaultValue;
+
+        Setting( final String defaultValue )
+        {
+            this.defaultValue = defaultValue == null ? "" : defaultValue;
+        }
+
+        private String getDefaultValue( )
+        {
+            return defaultValue;
+        }
+    }
+
+    enum FtpMode {
+        ftp,
+        ftps,
+    }
+
+    private final Map<Setting,String> settings;
+
+    private Settings( final Map<Setting, String> settings )
+    {
+        this.settings = settings;
+    }
+
+    static Settings readFromFile( final String filename) throws IOException {
+        final Properties properties = new Properties();
+        properties.load(new FileReader(filename));
+        final Map<Setting,String> returnMap = new HashMap<>(  );
+        for (final Setting setting : Setting.values() )
+        {
+            final String value = properties.getProperty( setting.name(), setting.getDefaultValue() );
+            returnMap.put( setting, value );
+        }
+        return new Settings( Collections.unmodifiableMap( returnMap ) );
+    }
+
+    public String getSetting( final Setting setting )
+    {
+        return settings.get( setting );
+    }
+
+    public boolean isFtpEnabled() {
+        final String value = settings.get( Setting.ftpSite );
+        return !StringUtil.isEmpty( value );
+    }
+}

data-service/src/main/java/password/pwm/receiver/Status.java

@@ -0,0 +1,38 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import lombok.Getter;
+import lombok.Setter;
+
+import java.time.Instant;
+
+@Getter
+@Setter
+public class Status {
+    private String errorState;
+    private String lastFtpStatus;
+    private Instant lastFtpIngest;
+    private int lastFtpFilesRead;
+}

data-service/src/main/java/password/pwm/receiver/Storage.java

@@ -0,0 +1,188 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import jetbrains.exodus.ArrayByteIterable;
+import jetbrains.exodus.ByteIterable;
+import jetbrains.exodus.bindings.StringBinding;
+import jetbrains.exodus.env.Cursor;
+import jetbrains.exodus.env.Environment;
+import jetbrains.exodus.env.EnvironmentConfig;
+import jetbrains.exodus.env.Environments;
+import jetbrains.exodus.env.Store;
+import jetbrains.exodus.env.StoreConfig;
+import jetbrains.exodus.env.Transaction;
+import password.pwm.bean.TelemetryPublishBean;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.util.java.StringUtil;
+
+import java.io.File;
+import java.io.IOException;
+import java.time.Instant;
+import java.util.Iterator;
+
+public class Storage {
+    private final Environment environment;
+    private Store store;
+
+    public Storage(final Settings settings) throws IOException {
+        final String path = settings.getSetting( Settings.Setting.storagePath );
+        if (path == null) {
+            throw new IOException("data path is not specified!");
+        }
+
+        final File dataPath = new File(path);
+        if (!dataPath.exists()) {
+            throw new IOException("data path '" + dataPath + "' does not exist");
+        }
+
+        final File stoagePath = new File(dataPath.getAbsolutePath() + File.separator + "storage");
+        stoagePath.mkdir();
+
+        final EnvironmentConfig environmentConfig = new EnvironmentConfig();
+        environment = Environments.newInstance(stoagePath.getAbsolutePath(), environmentConfig);
+
+        environment.executeInTransaction(txn -> store
+                = environment.openStore("store1", StoreConfig.WITHOUT_DUPLICATES, txn));
+    }
+
+    public void store(final TelemetryPublishBean bean) {
+        if (bean == null) {
+            return;
+        }
+
+        final String instanceHash = bean.getInstanceHash();
+        if (instanceHash != null) {
+            final TelemetryPublishBean existingBean = get(instanceHash);
+            Instant existingTimestamp = null;
+            if (existingBean != null) {
+                existingTimestamp = existingBean.getTimestamp();
+            }
+            if (existingTimestamp == null || existingTimestamp.isBefore(bean.getTimestamp())) {
+                put(bean);
+            }
+        }
+    }
+
+    public Iterator<TelemetryPublishBean> iterator() {
+        return new InnerIterator();
+    }
+
+    private boolean put(final TelemetryPublishBean value) {
+        return environment.computeInTransaction(transaction -> {
+            final ByteIterable k = StringBinding.stringToEntry(value.getInstanceHash());
+            final ByteIterable v = StringBinding.stringToEntry(JsonUtil.serialize(value));
+            return store.put(transaction,k,v);
+        });
+    }
+
+    private TelemetryPublishBean get(final String hash) {
+        return environment.computeInTransaction(transaction -> {
+            final ByteIterable k = StringBinding.stringToEntry(hash);
+            final ByteIterable v = store.get(transaction,k);
+            if (v != null) {
+                final String string = StringBinding.entryToString(new ArrayByteIterable(v));
+                if (!StringUtil.isEmpty(string)) {
+                    return JsonUtil.deserialize(string, TelemetryPublishBean.class);
+                }
+            }
+            return null;
+        });
+    }
+
+    public void close() {
+        store.getEnvironment().close();
+    }
+
+    public long count() {
+        return environment.computeInTransaction( transaction -> store.count( transaction ) );
+    }
+
+    private class InnerIterator implements AutoCloseable,Iterator {
+        private final Transaction transaction;
+        private final Cursor cursor;
+
+        private boolean closed;
+        private String nextValue = "";
+
+        InnerIterator() {
+            this.transaction = environment.beginReadonlyTransaction();
+            this.cursor = store.openCursor(transaction);
+            doNext();
+        }
+
+        private void doNext() {
+            try {
+                if (closed) {
+                    return;
+                }
+
+                if (!cursor.getNext()) {
+                    close();
+                    return;
+                }
+                final ByteIterable nextKey = cursor.getKey();
+                final String string = StringBinding.entryToString(new ArrayByteIterable(nextKey));
+
+                if (string == null || string.isEmpty()) {
+                    close();
+                    return;
+                }
+                nextValue = string;
+            } catch (Exception e) {
+                e.printStackTrace();
+                throw e;
+            }
+        }
+
+        @Override
+        public void close() {
+            if (closed) {
+                return;
+            }
+            cursor.close();
+            transaction.abort();
+            nextValue = null;
+            closed = true;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return !closed && nextValue != null;
+        }
+
+        @Override
+        public TelemetryPublishBean next() {
+            final String value = nextValue;
+            doNext();
+            return get(value);
+        }
+
+        @Override
+        public void remove() {
+            throw new UnsupportedOperationException("remove not supported");
+        }
+    }
+
+}

data-service/src/main/java/password/pwm/receiver/SummaryBean.java

@@ -0,0 +1,194 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import lombok.Builder;
+import lombok.Getter;
+import lombok.Value;
+import password.pwm.PwmAboutProperty;
+import password.pwm.bean.TelemetryPublishBean;
+import password.pwm.config.PwmSetting;
+import password.pwm.svc.stats.Statistic;
+import password.pwm.util.java.TimeDuration;
+
+import java.time.Duration;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+
+@Getter
+@Builder
+public class SummaryBean {
+    private int serverCount;
+    private Map<String,SiteSummary> siteSummary;
+    private Map<String,Integer> ldapVendorCount;
+    private Map<String,Integer> appServerCount;
+    private Map<String,Integer> settingCount;
+    private Map<String,Integer> statCount;
+    private Map<String,Integer> osCount;
+    private Map<String,Integer> dbCount;
+    private Map<String,Integer> javaCount;
+    private Map<String,Integer> ssprVersionCount;
+
+    static SummaryBean fromStorage(final Storage storage, final TimeDuration maxAge ) {
+
+        final String naText = "n/a";
+
+        int serverCount = 0;
+        final Map<String,SiteSummary> siteSummaryMap = new TreeMap<>();
+        final Map<String,Integer> ldapVendorCount = new TreeMap<>();
+        final Map<String,Integer> appServerCount = new TreeMap<>();
+        final Map<String,Integer> settingCount = new TreeMap<>();
+        final Map<String,Integer> statCount = new TreeMap<>();
+        final Map<String,Integer> osCount = new TreeMap<>();
+        final Map<String,Integer> dbCount = new TreeMap<>();
+        final Map<String,Integer> javaCount = new TreeMap<>();
+        final Map<String,Integer> ssprVersionCount = new TreeMap<>();
+
+        for (Iterator<TelemetryPublishBean> iterator = storage.iterator(); iterator.hasNext(); ) {
+            final TelemetryPublishBean bean = iterator.next();
+            final TimeDuration age = TimeDuration.fromCurrent( bean.getTimestamp() );
+
+            if (bean.getAbout() != null && age.isShorterThan( maxAge ) ) {
+                serverCount++;
+                final String hashID = bean.getInstanceHash();
+                final String ldapVendor = bean.getLdapVendorName() == null
+                        ? naText
+                        : bean.getLdapVendorName();
+
+                final String dbVendor = dbVendorName(bean);
+
+                final SiteSummary siteSummary = SiteSummary.builder()
+                        .description(bean.getSiteDescription())
+                        .version(bean.getVersionVersion())
+                        .installAge(TimeDuration.fromCurrent(bean.getInstallTime()).asDuration())
+                        .updateAge(TimeDuration.fromCurrent(bean.getTimestamp()).asDuration())
+                        .ldapVendor(ldapVendor)
+                        .osName(bean.getAbout().get(PwmAboutProperty.java_osName.name()))
+                        .osVersion(bean.getAbout().get(PwmAboutProperty.java_osVersion.name()))
+                        .servletName(bean.getAbout().get(PwmAboutProperty.java_appServerInfo.name()))
+                        .dbVendor(dbVendor)
+                        .appliance(Boolean.parseBoolean(bean.getAbout().get(PwmAboutProperty.app_mode_appliance.name())))
+                        .javaVm(javaVmInfo( bean, "n/a" ))
+                        .build();
+
+                siteSummaryMap.put(hashID, siteSummary);
+
+                incrementCounterMap(dbCount, dbVendor);
+
+                incrementCounterMap(ldapVendorCount, ldapVendor);
+
+                incrementCounterMap(appServerCount, siteSummary.getServletName());
+
+                incrementCounterMap(osCount, bean.getAbout().get(PwmAboutProperty.java_osName.name()));
+
+                incrementCounterMap(javaCount, siteSummary.getJavaVm());
+
+                incrementCounterMap(ssprVersionCount, siteSummary.getVersion());
+
+                for (final String settingKey : bean.getConfiguredSettings()) {
+                    final PwmSetting setting = PwmSetting.forKey(settingKey);
+                    if (setting != null) {
+                        final String description = setting.toMenuLocationDebug(null, null);
+                        incrementCounterMap(settingCount, description);
+                    }
+                }
+
+                for (final String statKey : bean.getStatistics().keySet()) {
+                    final Statistic statistic = Statistic.forKey(statKey);
+                    if (statistic != null) {
+                        if (statistic.getType() == Statistic.Type.INCREMENTOR) {
+                            final int count = Integer.parseInt(bean.getStatistics().get(statKey));
+                            incrementCounterMap(statCount, statistic.getLabel(null), count);
+                        }
+                    }
+                }
+            }
+        }
+
+
+        return SummaryBean.builder()
+                .serverCount(serverCount)
+                .siteSummary(siteSummaryMap)
+                .ldapVendorCount(ldapVendorCount)
+                .settingCount(settingCount)
+                .statCount(statCount)
+                .appServerCount(appServerCount)
+                .osCount(osCount)
+                .dbCount(dbCount)
+                .javaCount(javaCount)
+                .ssprVersionCount(ssprVersionCount)
+                .build();
+
+    }
+
+    private static void incrementCounterMap(final Map<String,Integer> map, final String key) {
+        incrementCounterMap(map, key, 1);
+    }
+
+    private static void incrementCounterMap(final Map<String,Integer> map, final String key, final int count) {
+        if (map.containsKey(key)) {
+            map.put(key, map.get(key) + count);
+        } else {
+            map.put(key, count);
+        }
+    }
+
+    private static String dbVendorName(final TelemetryPublishBean bean) {
+        String dbVendor = "n/a";
+        final Map<String,String> aboutMap = bean.getAbout();
+        if (aboutMap.get(PwmAboutProperty.database_databaseProductName.name()) != null) {
+            dbVendor = aboutMap.get(PwmAboutProperty.database_databaseProductName.name());
+
+            if (aboutMap.get(PwmAboutProperty.database_databaseProductVersion.name()) != null) {
+                dbVendor += "/" + aboutMap.get(PwmAboutProperty.database_databaseProductVersion.name());
+            }
+        }
+        return dbVendor;
+    }
+
+    private static String javaVmInfo(final TelemetryPublishBean bean, final String naText ) {
+        return bean.getAbout().getOrDefault( PwmAboutProperty.java_vmName.name(), naText )
+                + " ("
+                + bean.getAbout().getOrDefault( PwmAboutProperty.java_vmVendor.name(), naText )
+                + " ) "
+                + bean.getAbout().getOrDefault( PwmAboutProperty.java_vmVersion.name(), naText );
+    }
+
+    @Value
+    @Builder
+    public static class SiteSummary {
+        private String description;
+        private String version;
+        private Duration installAge;
+        private Duration updateAge;
+        private String ldapVendor;
+        private String osName;
+        private String osVersion;
+        private String servletName;
+        private String dbVendor;
+        private String javaVm;
+        private boolean appliance;
+    }
+}

data-service/src/main/java/password/pwm/receiver/TelemetryRestReceiver.java

@@ -0,0 +1,97 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import org.apache.commons.io.IOUtils;
+import password.pwm.PwmConstants;
+import password.pwm.bean.TelemetryPublishBean;
+import password.pwm.error.ErrorInformation;
+import password.pwm.error.PwmError;
+import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.i18n.Message;
+import password.pwm.util.java.JsonUtil;
+import password.pwm.ws.server.RestResultBean;
+
+import javax.servlet.ServletException;
+import javax.servlet.annotation.WebServlet;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringWriter;
+
+@WebServlet(
+        name="TelemetryRestReceiver",
+        urlPatterns={
+                "/telemetry",
+        }
+)
+
+public class TelemetryRestReceiver extends HttpServlet {
+    @Override
+    protected void doPost(final HttpServletRequest req, final HttpServletResponse resp)
+            throws ServletException, IOException
+    {
+        try {
+            resp.setHeader("Content","application/json");
+            final String input = readRequestBodyAsString(req, 1024 * 1024);
+            final TelemetryPublishBean telemetryPublishBean = JsonUtil.deserialize(input, TelemetryPublishBean.class);
+            final Storage stoage = ContextManager.getContextManager(this.getServletContext()).getApp().getStorage();
+            stoage.store(telemetryPublishBean);
+            resp.getWriter().print(RestResultBean.forSuccessMessage(null, null, null, Message.Success_Unknown).toJson());
+        } catch (PwmUnrecoverableException e) {
+            resp.getWriter().print(RestResultBean.fromError(e.getErrorInformation()).toJson());
+        } catch (Exception e) {
+            final RestResultBean restResultBean = RestResultBean.fromError(new ErrorInformation(PwmError.ERROR_UNKNOWN, e.getMessage()));
+            resp.getWriter().print(restResultBean.toJson());
+        }
+    }
+
+    private static String readRequestBodyAsString(final HttpServletRequest req, final int maxChars)
+            throws IOException, PwmUnrecoverableException
+    {
+        final StringWriter stringWriter = new StringWriter();
+        final Reader readerStream = new InputStreamReader(
+                req.getInputStream(),
+                PwmConstants.DEFAULT_CHARSET
+        );
+
+        try {
+            IOUtils.copy(readerStream, stringWriter);
+        } catch (Exception e) {
+            final String errorMsg = "error reading request body stream: " + e.getMessage();
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN,errorMsg));
+        } finally {
+            IOUtils.closeQuietly(readerStream);
+        }
+
+        final String stringValue = stringWriter.toString();
+        if (stringValue.length() > maxChars) {
+            throw new PwmUnrecoverableException(new ErrorInformation(PwmError.ERROR_UNKNOWN,"input request body is to big, size=" + stringValue.length() + ", max=" + maxChars));
+        }
+        return stringValue;
+    }
+}

data-service/src/main/java/password/pwm/receiver/TelemetryViewerServlet.java

@@ -0,0 +1,71 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2017 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+package password.pwm.receiver;
+
+import password.pwm.util.java.StringUtil;
+import password.pwm.util.java.TimeDuration;
+
+import javax.servlet.ServletException;
+import javax.servlet.annotation.WebServlet;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+@WebServlet(
+        name="TelemetryViewer",
+        urlPatterns={
+                "/viewer",
+        }
+)
+public class TelemetryViewerServlet extends HttpServlet {
+    private static final String PARAM_DAYS = "days";
+
+    public static String SUMMARY_ATTR = "SummaryBean";
+
+    @Override
+    protected void doGet(final HttpServletRequest req, final HttpServletResponse resp) throws ServletException, IOException
+    {
+        final String daysString = req.getParameter( PARAM_DAYS );
+        final int days = StringUtil.isEmpty( daysString ) ? 30 : Integer.parseInt( daysString );
+        final ContextManager contextManager = ContextManager.getContextManager(req.getServletContext());
+        final PwmReceiverApp app = contextManager.getApp();
+
+        {
+            final String errorState = app.getStatus().getErrorState();
+            if (!StringUtil.isEmpty(errorState)) {
+                resp.sendError(500, errorState);
+                final String htmlBody = "<html>Error: " + errorState + "</html>";
+                resp.getWriter().print(htmlBody);
+                return;
+            }
+        }
+
+        final Storage storage = app.getStorage();
+        final SummaryBean summaryBean = SummaryBean.fromStorage(storage, new TimeDuration(days, TimeUnit.DAYS ) );
+        req.setAttribute(SUMMARY_ATTR, summaryBean);
+        req.getServletContext().getRequestDispatcher("/WEB-INF/jsp/telemetry-viewer.jsp").forward(req,resp);
+    }
+}

data-service/src/main/webapp/META-INF/context.xml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ Password Management Servlets (PWM)
+  ~ http://www.pwm-project.org
+  ~
+  ~ Copyright (c) 2006-2009 Novell, Inc.
+  ~ Copyright (c) 2009-2017 The PWM Project
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or
+  ~ (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+  ~
+  -->
+
+<Context tldValidation="false" unloadDelay="30000" useHttpOnly="true">
+
+</Context>

data-service/src/main/webapp/WEB-INF/jsp/telemetry-viewer.jsp

@@ -0,0 +1,207 @@
+<%@ page import="password.pwm.config.PwmSetting" %>
+<%@ page import="password.pwm.receiver.SummaryBean" %>
+<%@ page import="password.pwm.receiver.TelemetryViewerServlet" %>
+<%@ page import="org.joda.time.DateTime" %>
+<%@ page import="java.time.format.DateTimeFormatter" %>
+<%@ page import="java.time.Instant" %>
+<%@ page import="java.time.LocalDateTime" %>
+<%@ page import="java.time.format.FormatStyle" %>
+<%@ page import="password.pwm.receiver.PwmReceiverApp" %>
+<%@ page import="password.pwm.receiver.ContextManager" %>
+<%--
+  ~ Password Management Servlets (PWM)
+  ~ http://www.pwm-project.org
+  ~
+  ~ Copyright (c) 2006-2009 Novell, Inc.
+  ~ Copyright (c) 2009-2017 The PWM Project
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or
+  ~ (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+  ~
+  --%>
+
+<!DOCTYPE html>
+<%@ page contentType="text/html" %>
+<% SummaryBean summaryBean = (SummaryBean)request.getAttribute(TelemetryViewerServlet.SUMMARY_ATTR); %>
+<% PwmReceiverApp app = ContextManager.getContextManager(request.getServletContext()).getApp(); %>
+<html>
+<head>
+    <title>Telemetry Data</title>
+</head>
+<body>
+<div>
+    Current Time: <%=Instant.now().toString()%>
+    <br/>
+    <% if (app.getSettings().isFtpEnabled()) {%>
+    <% Instant lastIngest = app.getStatus().getLastFtpIngest(); %>
+    Last FTP Ingestion: <%= lastIngest == null ? "n/a" : lastIngest.toString()%>
+    <br/>
+    Last FTP Status: <%= app.getStatus().getLastFtpStatus()%>
+    <br/>
+    FTP Files On Server: <%= app.getStatus().getLastFtpFilesRead()%>
+    <br/>
+    <% } %>
+    Servers Registered: <%= app.getStorage().count() %>
+    <br/>
+    Servers Shown: <%= summaryBean.getServerCount() %>
+    <br/>
+    <br/>
+
+    <form method="get">
+        <label>Servers that have sent data in last number of days
+            <input type="number" name="days" id="days" value="30" max="3650" min="1">
+        </label>
+        <button type="submit">Update</button>
+    </form>
+
+    <h2>Versions</h2>
+    <table border="1">
+        <tr>
+            <td><b>Version</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String version : summaryBean.getSsprVersionCount().keySet()) { %>
+        <tr>
+            <td><%=version%></td>
+            <td><%=summaryBean.getSsprVersionCount().get(version)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>LDAP Vendors</h2>
+    <table border="1">
+        <tr>
+            <td><b>Ldap</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String ldapVendor : summaryBean.getLdapVendorCount().keySet()) { %>
+        <tr>
+            <td><%=ldapVendor%></td>
+            <td><%=summaryBean.getLdapVendorCount().get(ldapVendor)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>App Servers</h2>
+    <table border="1">
+        <tr>
+            <td><b>App Server Info</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String appServerInfo : summaryBean.getAppServerCount().keySet()) { %>
+        <tr>
+            <td><%=appServerInfo%></td>
+            <td><%=summaryBean.getAppServerCount().get(appServerInfo)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>OS Vendors</h2>
+    <table border="1">
+        <tr>
+            <td><b>OS Vendor</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String osName : summaryBean.getOsCount().keySet()) { %>
+        <tr>
+            <td><%=osName%></td>
+            <td><%=summaryBean.getOsCount().get(osName)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>DB Vendors</h2>
+    <table border="1">
+        <tr>
+            <td><b>DB Vendor</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String dbName : summaryBean.getDbCount().keySet()) { %>
+        <tr>
+            <td><%=dbName%></td>
+            <td><%=summaryBean.getDbCount().get(dbName)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>Java VMs</h2>
+    <table border="1">
+        <tr>
+            <td><b>Java VM</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String javaName : summaryBean.getJavaCount().keySet()) { %>
+        <tr>
+            <td><%=javaName%></td>
+            <td><%=summaryBean.getJavaCount().get(javaName)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>Settings</h2>
+    <table border="1">
+        <tr>
+            <td><b>Setting</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String setting: summaryBean.getSettingCount().keySet()) { %>
+        <tr>
+            <td><%=setting%></td>
+            <td><%=summaryBean.getSettingCount().get(setting)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <h2>Statistics</h2>
+    <table border="1">
+        <tr>
+            <td><b>Statistic</b></td>
+            <td><b>Count</b></td>
+        </tr>
+        <% for (final String statistic: summaryBean.getStatCount().keySet()) { %>
+        <tr>
+            <td><%=statistic%></td>
+            <td><%=summaryBean.getStatCount().get(statistic)%></td>
+        </tr>
+        <% } %>
+    </table>
+    <br/>
+    <h2>Summary Data</h2>
+    <table border="1">
+        <tr>
+            <td><b>SiteHash</b></td>
+            <td><b>Description</b></td>
+            <td><b>Version</b></td>
+            <td><b>Installed</b></td>
+            <td><b>Last Updated</b></td>
+            <td><b>Ldap</b></td>
+            <td><b>OS Name</b></td>
+            <td><b>OS Version</b></td>
+            <td><b>Servlet Name</b></td>
+            <td><b>DB Vendor</b></td>
+            <td><b>Appliance</b></td>
+        </tr>
+        <% for (final String hashID : summaryBean.getSiteSummary().keySet()) { %>
+        <% SummaryBean.SiteSummary siteSummary = summaryBean.getSiteSummary().get(hashID); %>
+        <tr>
+            <td style="max-width: 500px; overflow: auto"><%=hashID%></td>
+            <td><%=siteSummary.getDescription()%></td>
+            <td><%=siteSummary.getVersion()%></td>
+            <td><%=siteSummary.getInstallAge()%></td>
+            <td><%=siteSummary.getUpdateAge()%></td>
+            <td><%=siteSummary.getLdapVendor()%></td>
+            <td><%=siteSummary.getOsName()%></td>
+            <td><%=siteSummary.getOsVersion()%></td>
+            <td><%=siteSummary.getServletName()%></td>
+            <td><%=siteSummary.getDbVendor()%></td>
+            <td><%=siteSummary.isAppliance()%></td>
+        </tr>
+        <% } %>
+    </table>
+</div>
+</body>
+</html>

data-service/src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Password Management Servlets (PWM)
+  ~ http://www.pwm-project.org
+  ~
+  ~ Copyright (c) 2006-2009 Novell, Inc.
+  ~ Copyright (c) 2009-2017 The PWM Project
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or
+  ~ (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+  ~
+  -->
+
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://java.sun.com/xml/ns/javaee"
+         xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
+         id="PWM" version="3.0">
+    <display-name>PWM Receiver</display-name>
+    <!-- <distributable/> Clustering/Session replication is not supported -->
+    <description>Password Management Servlet</description>
+    <context-param>
+        <description>
+            Explicit location of application path working directory or the literal value "unspecified".  See the environment documentation at /public/reference/environment.jsp for more information.
+        </description>
+        <param-name>applicationPath</param-name>
+        <param-value>unspecified</param-value>
+    </context-param>
+    <welcome-file-list>
+        <welcome-file>index.jsp</welcome-file>
+    </welcome-file-list>
+    <session-config>
+        <session-timeout>5</session-timeout>
+        <cookie-config>
+            <http-only>true</http-only>
+        </cookie-config>
+    </session-config>
+    <jsp-config>
+        <jsp-property-group>
+            <url-pattern>*.jsp</url-pattern>
+            <trim-directive-whitespaces>true</trim-directive-whitespaces>
+        </jsp-property-group>
+        <jsp-property-group>
+            <url-pattern>*.jsp</url-pattern>
+            <page-encoding>UTF-8</page-encoding>
+        </jsp-property-group>
+    </jsp-config>
+</web-app>

data-service/src/main/webapp/index.jsp

@@ -0,0 +1,32 @@
+<%--
+  ~ Password Management Servlets (PWM)
+  ~ http://www.pwm-project.org
+  ~
+  ~ Copyright (c) 2006-2009 Novell, Inc.
+  ~ Copyright (c) 2009-2017 The PWM Project
+  ~
+  ~ This program is free software; you can redistribute it and/or modify
+  ~ it under the terms of the GNU General Public License as published by
+  ~ the Free Software Foundation; either version 2 of the License, or
+  ~ (at your option) any later version.
+  ~
+  ~ This program is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  ~ GNU General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU General Public License
+  ~ along with this program; if not, write to the Free Software
+  ~ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+  ~
+  --%>
+
+<!DOCTYPE html>
+<%@ page language="java" session="true" isThreadSafe="true"
+         contentType="text/html" %>
+<html>
+<body>
+<div>html-pwm-receiver
+</div>
+</body>
+</html>

onejar/pom.xml

@@ -81,7 +81,7 @@
                     </descriptors>
                     <archive>
                         <manifestEntries>
-                            <Main-Class>password.pwm.TomcatOneJarMain</Main-Class>
+                            <Main-Class>password.pwm.onejar.TomcatOneJarMain</Main-Class>
                             <Implementation-Title>${project.name}</Implementation-Title>
                             <Implementation-Version>${project.version}</Implementation-Version>
                             <Implementation-Vendor>${project.organization.name}</Implementation-Vendor>

onejar/src/main/java/password/pwm/Argument.java → onejar/src/main/java/password/pwm/onejar/Argument.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;

onejar/src/main/java/password/pwm/ArgumentParser.java → onejar/src/main/java/password/pwm/onejar/ArgumentParser.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.DefaultParser;

onejar/src/main/java/password/pwm/ArgumentParserException.java → onejar/src/main/java/password/pwm/onejar/ArgumentParserException.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 public class ArgumentParserException extends Exception
 {

onejar/src/main/java/password/pwm/Resource.java → onejar/src/main/java/password/pwm/onejar/Resource.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 import java.util.ResourceBundle;
 

onejar/src/main/java/password/pwm/TomcatConfig.java → onejar/src/main/java/password/pwm/onejar/TomcatConfig.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 import java.io.File;
 import java.io.InputStream;

onejar/src/main/java/password/pwm/TomcatOneJarException.java → onejar/src/main/java/password/pwm/onejar/TomcatOneJarException.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 public class TomcatOneJarException extends Exception
 {

onejar/src/main/java/password/pwm/TomcatOneJarMain.java → onejar/src/main/java/password/pwm/onejar/TomcatOneJarMain.java

@@ -20,7 +20,7 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-package password.pwm;
+package password.pwm.onejar;
 
 import org.apache.catalina.LifecycleException;
 import org.apache.catalina.connector.Connector;

onejar/src/main/java/password/pwm/onejar/WebServer.java

@@ -0,0 +1,27 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.onejar;
+
+public interface WebServer
+{
+}

server/pom.xml

@@ -120,7 +120,7 @@
                     <dependency>
                         <groupId>com.github.spotbugs</groupId>
                         <artifactId>spotbugs</artifactId>
-                        <version>3.1.3</version>
+                        <version>3.1.4</version>
                     </dependency>
                 </dependencies>
                 <configuration>
@@ -200,24 +200,6 @@
                 <configuration>
                     <source>${maven.compiler.source}</source>
                     <target>${maven.compiler.target}</target>
-
-                    <!-- following allows lombok processor to execute on jdk9+ -->
-                    <showDeprecation>true</showDeprecation>
-                    <showWarnings>true</showWarnings>
-                    <fork>true</fork>
-                    <compilerargs>
-                        <arg>-Werror</arg>
-                        <arg>-Xlint:all</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED</arg>
-                        <arg>-J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED</arg>
-                    </compilerargs>
                 </configuration>
             </plugin>
             <plugin>
@@ -601,7 +583,7 @@
         <dependency>
             <groupId>com.github.spotbugs</groupId>
             <artifactId>spotbugs-annotations</artifactId>
-            <version>3.1.3</version>
+            <version>3.1.4</version>
             <scope>provided</scope>
         </dependency>
 

server/src/build/checkstyle-import.xml

@@ -74,6 +74,7 @@
     <allow pkg="javax.net"/>
     <allow pkg="javax.crypto"/>
     <allow pkg="javax.mail"/>
+    <allow class="com.sun.mail.smtp.SMTPSendFailedException"/>
     <allow pkg="org.xeustechnologies"/>
     <allow pkg="net.glxn"/>
     <allow pkg="org.webjars"/>

server/src/main/java/password/pwm/AppProperty.java

@@ -307,6 +307,7 @@ public enum AppProperty
     SECURITY_DEFAULT_EPHEMERAL_HASH_ALG             ( "security.defaultEphemeralHashAlg" ),
     SEEDLIST_BUILTIN_PATH                           ( "seedlist.builtin.path" ),
     SMTP_SUBJECT_ENCODING_CHARSET                   ( "smtp.subjectEncodingCharset" ),
+    SMTP_RETRYABLE_SEND_RESPONSE_STATUSES           ( "smtp.retryableSendResponseStatus" ),
     TOKEN_CLEANER_INTERVAL_SECONDS                  ( "token.cleaner.intervalSeconds" ),
     TOKEN_MASK_EMAIL_REGEX                          ( "token.mask.email.regex" ),
     TOKEN_MASK_EMAIL_REPLACE                        ( "token.mask.email.replace" ),

server/src/main/java/password/pwm/config/value/ActionValue.java

@@ -128,10 +128,17 @@ public class ActionValue extends AbstractValue implements StoredValue
                             final List<ActionConfiguration.WebAction> clonedWebActions = new ArrayList<>();
                             for ( final ActionConfiguration.WebAction webAction : value.getWebActions() )
                             {
+                                // add success status if empty list
+                                final List<Integer> successStatus = JavaHelper.isEmpty( webAction.getSuccessStatus() )
+                                        ? Collections.singletonList( 200 )
+                                        : webAction.getSuccessStatus();
+
+                                // decrypt pw
                                 try
                                 {
                                     clonedWebActions.add( webAction.toBuilder()
                                             .password( decryptPwValue( webAction.getPassword(), pwmSecurityKey ) )
+                                            .successStatus( successStatus )
                                             .build() );
                                 }
                                 catch ( PwmOperationalException e )
@@ -265,27 +272,31 @@ public class ActionValue extends AbstractValue implements StoredValue
             for ( final ActionConfiguration.WebAction webAction : actionConfiguration.getWebActions() )
             {
                 sb.append( "\n   WebServiceAction: " );
-                sb.append( "method=" ).append( webAction.getMethod() );
-                sb.append( " url=" ).append( webAction.getUrl() );
-                sb.append( " headers=" ).append( JsonUtil.serializeMap( webAction.getHeaders() ) );
-                sb.append( " username=" ).append( webAction.getUsername() );
-                sb.append( " password=" ).append(
+                sb.append( "\n    method=" ).append( webAction.getMethod() );
+                sb.append( "\n    url=" ).append( webAction.getUrl() );
+                sb.append( "\n    headers=" ).append( JsonUtil.serializeMap( webAction.getHeaders() ) );
+                sb.append( "\n    username=" ).append( webAction.getUsername() );
+                sb.append( "\n    password=" ).append(
                         StringUtil.isEmpty( webAction.getPassword() )
                                 ? ""
                                 : PwmConstants.LOG_REMOVED_VALUE_REPLACEMENT
                 );
+                if ( !JavaHelper.isEmpty( webAction.getSuccessStatus() ) )
+                {
+                    sb.append( "\n    successStatus=" ).append( StringUtil.collectionToString( webAction.getSuccessStatus() ) );
+                }
                 if ( StringUtil.isEmpty( webAction.getBody() ) )
                 {
-                    sb.append( " body=" ).append( webAction.getBody() );
+                    sb.append( "\n    body=" ).append( webAction.getBody() );
                 }
             }
 
             for ( final ActionConfiguration.LdapAction ldapAction : actionConfiguration.getLdapActions() )
             {
                 sb.append( "\n   LdapAction: " );
-                sb.append( "method=" ).append( ldapAction.getLdapMethod() );
-                sb.append( " attribute=" ).append( ldapAction.getAttributeName() );
-                sb.append( " value=" ).append( ldapAction.getAttributeValue() );
+                sb.append( "\n    method=" ).append( ldapAction.getLdapMethod() );
+                sb.append( "\n    attribute=" ).append( ldapAction.getAttributeName() );
+                sb.append( "\n    value=" ).append( ldapAction.getAttributeValue() );
             }
             counter++;
             if ( counter != values.size() )

server/src/main/java/password/pwm/config/value/EmailValue.java

@@ -124,6 +124,8 @@ public class EmailValue extends AbstractValue implements StoredValue
 
     public List<String> validateValue( final PwmSetting pwmSetting )
     {
+        final int maxBodyChars = 500_000;
+
         if ( pwmSetting.isRequired() )
         {
             if ( values == null || values.isEmpty() || values.values().iterator().next() == null )
@@ -151,6 +153,18 @@ public class EmailValue extends AbstractValue implements StoredValue
             {
                 return Collections.singletonList( "plain body field is required" + ( loopLocale.length() > 0 ? " for locale " + loopLocale : "" ) );
             }
+
+            if ( emailItemBean.getBodyPlain() == null || emailItemBean.getBodyPlain().length() > maxBodyChars )
+            {
+                return Collections.singletonList( "plain body field is too large" + ( loopLocale.length() > 0 ? " for locale " + loopLocale : "" )
+                        + ", chars=" + emailItemBean.getBodyPlain().length() + ", max=" + maxBodyChars );
+            }
+
+            if ( emailItemBean.getBodyHtml() == null || emailItemBean.getBodyHtml().length() > maxBodyChars )
+            {
+                return Collections.singletonList( "html body field is too large" + ( loopLocale.length() > 0 ? " for locale " + loopLocale : "" )
+                        + ", chars=" + emailItemBean.getBodyHtml().length() + ", max=" + maxBodyChars );
+            }
         }
 
         return Collections.emptyList();

server/src/main/java/password/pwm/config/value/data/ActionConfiguration.java

@@ -74,6 +74,9 @@ public class ActionConfiguration implements Serializable
 
         @Builder.Default
         private List<X509Certificate> certificates = Collections.emptyList();
+
+        @Builder.Default
+        private List<Integer> successStatus = Collections.singletonList( 200 );
     }
 
     @Value

server/src/main/java/password/pwm/http/HttpHeader.java

@@ -28,33 +28,34 @@ import password.pwm.util.java.StringUtil;
 
 public enum HttpHeader
 {
+    Authorization( "Authorization", Property.Sensitive ),
     Accept( "Accept" ),
+    AcceptEncoding( "Accept-Encoding" ),
+    AcceptLanguage( "Accept-Language" ),
+    CacheControl( "Cache-Control" ),
     Connection( "Connection" ),
-    Content_Type( "Content-Type" ),
-    Content_Encoding( "Content-Encoding" ),
-    Location( "Location" ),
+    ContentEncoding( "Content-Encoding" ),
+    ContentDisposition( "content-disposition" ),
+    ContentLanguage( "Content-Language" ),
+    ContentLength( "Content-Length" ),
     ContentSecurityPolicy( "Content-Security-Policy" ),
+    ContentTransferEncoding( "Content-Transfer-Encoding" ),
+    ContentType( "Content-Type" ),
+    ETag( "ETag" ),
+    Expires( "Expires" ),
     If_None_Match( "If-None-Match" ),
+    Location( "Location" ),
+    Origin( "Origin" ),
+    Referer( "Referer" ),
     Server( "Server" ),
-    Cache_Control( "Cache-Control" ),
-    WWW_Authenticate( "WWW-Authenticate" ),
-    ContentDisposition( "content-disposition" ),
-    ContentTransferEncoding( "Content-Transfer-Encoding" ),
-    Content_Language( "Content-Language" ),
-    Accept_Encoding( "Accept-Encoding" ),
-    Accept_Language( "Accept-Language" ),
-    Authorization( "Authorization", Property.Sensitive ),
     UserAgent( "User-Agent" ),
-    Referer( "Referer" ),
-    Origin( "Origin" ),
+    WWW_Authenticate( "WWW-Authenticate" ),
+    XContentTypeOptions( "X-Content-Type-Options" ),
     XForwardedFor( "X-Forwarded-For" ),
-    ETag( "ETag" ),
-    Expires( "Expires" ),
-
     XFrameOptions( "X-Frame-Options" ),
-    XContentTypeOptions( "X-Content-Type-Options" ),
     XXSSProtection( "X-XSS-Protection" ),
 
+
     XAmb( "X-" + PwmConstants.PWM_APP_NAME + "-Amb" ),
     XVersion( "X-" + PwmConstants.PWM_APP_NAME + "-Version" ),
     XInstance( "X-" + PwmConstants.PWM_APP_NAME + "-Instance" ),

server/src/main/java/password/pwm/http/bean/ConfigGuideBean.java

@@ -23,6 +23,7 @@
 package password.pwm.http.bean;
 
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import password.pwm.config.option.SessionBeanMode;
 import password.pwm.config.value.FileValue;
 import password.pwm.http.servlet.configguide.ConfigGuideForm;
@@ -37,6 +38,7 @@ import java.util.Map;
 import java.util.Set;
 
 @Data
+@EqualsAndHashCode( callSuper = false )
 public class ConfigGuideBean extends PwmSessionBean
 {
 

server/src/main/java/password/pwm/http/bean/ForgottenPasswordBean.java

@@ -26,6 +26,7 @@ import com.google.gson.annotations.SerializedName;
 import com.novell.ldapchai.cr.ChallengeSet;
 import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import lombok.Value;
 import password.pwm.VerificationMethodSystem;
 import password.pwm.bean.TokenDestinationItem;
@@ -46,6 +47,7 @@ import java.util.Set;
  * @author Jason D. Rivard
  */
 @Data
+@EqualsAndHashCode( callSuper = false )
 public class ForgottenPasswordBean extends PwmSessionBean
 {
 

server/src/main/java/password/pwm/http/bean/UpdateProfileBean.java

@@ -24,6 +24,7 @@ package password.pwm.http.bean;
 
 import com.google.gson.annotations.SerializedName;
 import lombok.Data;
+import lombok.EqualsAndHashCode;
 import password.pwm.config.option.SessionBeanMode;
 
 import java.util.Arrays;
@@ -34,6 +35,7 @@ import java.util.Map;
 import java.util.Set;
 
 @Data
+@EqualsAndHashCode( callSuper = false )
 public class UpdateProfileBean extends PwmSessionBean
 {
 

server/src/main/java/password/pwm/http/filter/RequestInitializationFilter.java

@@ -369,7 +369,7 @@ public class RequestInitializationFilter implements Filter
         final boolean includeContentLanguage = Boolean.parseBoolean( config.readAppProperty( AppProperty.HTTP_HEADER_SEND_CONTENT_LANGUAGE ) );
         if ( includeContentLanguage )
         {
-            resp.setHeader( HttpHeader.Content_Language, pwmRequest.getLocale().toLanguageTag() );
+            resp.setHeader( HttpHeader.ContentLanguage, pwmRequest.getLocale().toLanguageTag() );
         }
 
         addStaticResponseHeaders( pwmApplication, resp.getHttpServletResponse() );
@@ -454,7 +454,7 @@ public class RequestInitializationFilter implements Filter
             ) );
         }
 
-        resp.setHeader( HttpHeader.Cache_Control.getHttpName(), "no-cache, no-store, must-revalidate, proxy-revalidate" );
+        resp.setHeader( HttpHeader.CacheControl.getHttpName(), "no-cache, no-store, must-revalidate, proxy-revalidate" );
     }
 
 

server/src/main/java/password/pwm/http/servlet/ClientApiServlet.java

@@ -162,7 +162,7 @@ public class ClientApiServlet extends ControlledPwmServlet
 
         pwmRequest.getPwmResponse().setHeader( HttpHeader.ETag, eTagValue );
         pwmRequest.getPwmResponse().setHeader( HttpHeader.Expires, String.valueOf( System.currentTimeMillis() + ( maxCacheAgeSeconds * 1000 ) ) );
-        pwmRequest.getPwmResponse().setHeader( HttpHeader.Cache_Control, "public, max-age=" + maxCacheAgeSeconds );
+        pwmRequest.getPwmResponse().setHeader( HttpHeader.CacheControl, "public, max-age=" + maxCacheAgeSeconds );
 
         final AppData appData = makeAppData(
                 pwmRequest.getPwmApplication(),
@@ -188,7 +188,7 @@ public class ClientApiServlet extends ControlledPwmServlet
 
         pwmRequest.getPwmResponse().setHeader( HttpHeader.ETag, eTagValue );
         pwmRequest.getPwmResponse().setHeader( HttpHeader.Expires, String.valueOf( System.currentTimeMillis() + ( maxCacheAgeSeconds * 1000 ) ) );
-        pwmRequest.getPwmResponse().setHeader( HttpHeader.Cache_Control, "public, max-age=" + maxCacheAgeSeconds );
+        pwmRequest.getPwmResponse().setHeader( HttpHeader.CacheControl, "public, max-age=" + maxCacheAgeSeconds );
 
         try
         {

server/src/main/java/password/pwm/http/servlet/command/CommandServlet.java

@@ -125,7 +125,7 @@ public abstract class CommandServlet extends ControlledPwmServlet
         pwmRequest.validatePwmFormID();
         if ( !pwmRequest.getPwmResponse().isCommitted() )
         {
-            pwmRequest.getPwmResponse().setHeader( HttpHeader.Cache_Control, "no-cache, no-store, must-revalidate" );
+            pwmRequest.getPwmResponse().setHeader( HttpHeader.CacheControl, "no-cache, no-store, must-revalidate" );
             pwmRequest.getPwmResponse().setContentType( HttpContentType.plain );
         }
         return ProcessStatus.Halt;
@@ -173,7 +173,7 @@ public abstract class CommandServlet extends ControlledPwmServlet
         LOGGER.debug( "pageLeaveNotice indicated at " + pageLeaveNoticeTime.toString() + ", referer=" + referrer );
         if ( !pwmRequest.getPwmResponse().isCommitted() )
         {
-            pwmRequest.getPwmResponse().setHeader( HttpHeader.Cache_Control, "no-cache, no-store, must-revalidate" );
+            pwmRequest.getPwmResponse().setHeader( HttpHeader.CacheControl, "no-cache, no-store, must-revalidate" );
             pwmRequest.getPwmResponse().setContentType( HttpContentType.plain );
         }
         return ProcessStatus.Halt;

server/src/main/java/password/pwm/http/servlet/configmanager/DebugItemGenerator.java

@@ -38,6 +38,7 @@ import password.pwm.http.servlet.admin.UserDebugDataBean;
 import password.pwm.http.servlet.admin.UserDebugDataReader;
 import password.pwm.ldap.LdapDebugDataGenerator;
 import password.pwm.svc.PwmService;
+import password.pwm.svc.cache.CacheService;
 import password.pwm.svc.cluster.ClusterService;
 import password.pwm.util.LDAPPermissionCalculator;
 import password.pwm.util.java.FileSystemUtility;
@@ -100,7 +101,8 @@ public class DebugItemGenerator
             LocalDBDebugGenerator.class,
             SessionDataGenerator.class,
             LdapRecentUserDebugGenerator.class,
-            ClusterInfoDebugGenerator.class
+            ClusterInfoDebugGenerator.class,
+            CacheServiceDebugItemGenerator.class
     ) );
 
     static void outputZipDebugFile(
@@ -687,6 +689,30 @@ public class DebugItemGenerator
         }
     }
 
+    static class CacheServiceDebugItemGenerator implements Generator
+    {
+        @Override
+        public String getFilename( )
+        {
+            return "cache-service-info.json";
+        }
+
+        @Override
+        public void outputItem(
+                final PwmApplication pwmApplication,
+                final PwmRequest pwmRequest,
+                final OutputStream outputStream
+        )
+                throws Exception
+        {
+            final CacheService cacheService = pwmApplication.getCacheService();
+
+            final Map<String, Serializable> debugOutput = new LinkedHashMap<>( cacheService.debugInfo() );
+            outputStream.write( JsonUtil.serializeMap( debugOutput, JsonUtil.Flag.PrettyPrint ).getBytes( PwmConstants.DEFAULT_CHARSET ) );
+        }
+
+    }
+
 
     interface Generator
     {

server/src/main/java/password/pwm/http/servlet/forgottenpw/RemoteVerificationMethod.java

@@ -143,8 +143,8 @@ public class RemoteVerificationMethod implements VerificationMethodSystem
         lastResponse = null;
 
         final Map<String, String> headers = new LinkedHashMap<>();
-        headers.put( HttpHeader.Content_Type.getHttpName(), HttpContentType.json.getHeaderValue() );
-        headers.put( HttpHeader.Accept_Language.getHttpName(), locale.toLanguageTag() );
+        headers.put( HttpHeader.ContentType.getHttpName(), HttpContentType.json.getHeaderValue() );
+        headers.put( HttpHeader.AcceptLanguage.getHttpName(), locale.toLanguageTag() );
 
         final RemoteVerificationRequestBean remoteVerificationRequestBean = new RemoteVerificationRequestBean();
         remoteVerificationRequestBean.setResponseSessionID( this.remoteSessionID );

server/src/main/java/password/pwm/http/servlet/oauth/OAuthMachine.java

@@ -253,7 +253,7 @@ public class OAuthMachine
             final Map<String, String> headers = new HashMap<>( );
             headers.put( HttpHeader.Authorization.getHttpName(),
                     new BasicAuthInfo( settings.getClientID(), settings.getSecret() ).toAuthHeader() );
-            headers.put( HttpHeader.Content_Type.getHttpName(), HttpContentType.form.getHeaderValue() );
+            headers.put( HttpHeader.ContentType.getHttpName(), HttpContentType.form.getHeaderValue() );
 
             pwmHttpClientRequest = new PwmHttpClientRequest( HttpMethod.POST, requestUrl, requestBody, headers );
         }

server/src/main/java/password/pwm/http/servlet/resource/ResourceFileServlet.java

@@ -226,7 +226,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
         {
             if ( contentType.startsWith( "text" ) || contentType.contains( "javascript" ) )
             {
-                final String acceptEncoding = pwmRequest.readHeaderValueAsString( HttpHeader.Accept_Encoding );
+                final String acceptEncoding = pwmRequest.readHeaderValueAsString( HttpHeader.AcceptEncoding );
                 acceptsGzip = acceptEncoding != null && accepts( acceptEncoding, "gzip" );
                 contentType += ";charset=UTF-8";
             }
@@ -254,7 +254,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
 
         // Initialize response.
         addExpirationHeaders( resourceConfiguration, response );
-        response.setHeader( "ETag", resourceConfiguration.getNonceValue() );
+        response.setHeader(  HttpHeader.ETag.getHttpName(), resourceConfiguration.getNonceValue() );
         response.setContentType( contentType );
 
         try
@@ -345,7 +345,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
                 if ( acceptsGzip )
                 {
                     final GZIPOutputStream gzipOutputStream = new GZIPOutputStream( tempOutputStream );
-                    headers.put( "Content-Encoding", "gzip" );
+                    headers.put( HttpHeader.ContentEncoding.getHttpName(), "gzip" );
                     copy( input, gzipOutputStream );
                     close( gzipOutputStream );
                 }
@@ -361,7 +361,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
             }
 
             final byte[] entity = tempOutputStream.toByteArray();
-            headers.put( "Content-Length", String.valueOf( entity.length ) );
+            headers.put( HttpHeader.ContentLength.getHttpName(), String.valueOf( entity.length ) );
             cacheEntry = new CacheEntry( entity, headers );
         }
         else
@@ -407,7 +407,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
             if ( acceptsGzip )
             {
                 // The browser accepts GZIP, so GZIP the content.
-                response.setHeader( "Content-Encoding", "gzip" );
+                response.setHeader( HttpHeader.ContentEncoding.getHttpName(), "gzip" );
                 output = new GZIPOutputStream( output );
             }
             else
@@ -416,7 +416,7 @@ public class ResourceFileServlet extends HttpServlet implements PwmServlet
                 // So only add it if there is no means of GZIP, else browser will hang.
                 if ( file.length() > 0 )
                 {
-                    response.setHeader( "Content-Length", String.valueOf( file.length() ) );
+                    response.setHeader( HttpHeader.ContentLength.getHttpName(), String.valueOf( file.length() ) );
                 }
             }
 

server/src/main/java/password/pwm/svc/cache/CacheDebugItem.java

@@ -0,0 +1,35 @@
+/*
+ * Password Management Servlets (PWM)
+ * http://www.pwm-project.org
+ *
+ * Copyright (c) 2006-2009 Novell, Inc.
+ * Copyright (c) 2009-2018 The PWM Project
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ */
+
+package password.pwm.svc.cache;
+
+import lombok.Value;
+
+import java.io.Serializable;
+
+@Value
+class CacheDebugItem implements Serializable
+{
+    private String key;
+    private String age;
+    private int chars;
+}

server/src/main/java/password/pwm/svc/cache/CacheService.java

@@ -34,9 +34,13 @@ import password.pwm.util.java.TimeDuration;
 import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.logging.PwmLogger;
 
+import java.io.Serializable;
 import java.time.Instant;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 public class CacheService implements PwmService
 {
@@ -110,6 +114,16 @@ public class CacheService implements PwmService
         return new ServiceInfoBean( Collections.emptyList() );
     }
 
+    public Map<String, Serializable> debugInfo( )
+    {
+        final Map<String, Serializable> debugInfo = new LinkedHashMap<>( );
+        debugInfo.put( "memory-statistics", memoryCacheStore.getCacheStoreInfo() );
+        debugInfo.put( "memory-items", new ArrayList<Serializable>( memoryCacheStore.getCacheDebugItems() ) );
+        debugInfo.put( "localdb-statistics", localDBCacheStore.getCacheStoreInfo() );
+        debugInfo.put( "localdb-items", new ArrayList<Serializable>( localDBCacheStore.getCacheDebugItems() ) );
+        return Collections.unmodifiableMap( debugInfo );
+    }
+
     public void put( final CacheKey cacheKey, final CachePolicy cachePolicy, final String payload )
             throws PwmUnrecoverableException
     {

server/src/main/java/password/pwm/svc/cache/CacheStore.java

@@ -25,6 +25,7 @@ package password.pwm.svc.cache;
 import password.pwm.error.PwmUnrecoverableException;
 
 import java.time.Instant;
+import java.util.List;
 
 public interface CacheStore
 {
@@ -35,4 +36,6 @@ public interface CacheStore
     CacheStoreInfo getCacheStoreInfo( );
 
     int itemCount( );
+
+    List<CacheDebugItem> getCacheDebugItems( );
 }

server/src/main/java/password/pwm/svc/cache/LocalDBCacheStore.java

@@ -30,8 +30,10 @@ import password.pwm.util.localdb.LocalDB;
 import password.pwm.util.localdb.LocalDBException;
 import password.pwm.util.logging.PwmLogger;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.TimerTask;
 import java.util.concurrent.ExecutorService;
@@ -226,4 +228,33 @@ public class LocalDBCacheStore implements CacheStore
         }
         return 0;
     }
+
+    @Override
+    public List<CacheDebugItem> getCacheDebugItems( )
+    {
+        final List<CacheDebugItem> items = new ArrayList<>();
+        try ( LocalDB.LocalDBIterator<String> iter = localDB.iterator( DB ) )
+        {
+            while ( iter.hasNext() )
+            {
+                final String nextKey = iter.next();
+                final String storedValue = localDB.get( DB, nextKey );
+                if ( storedValue != null )
+                {
+                    final CacheValueWrapper valueWrapper = JsonUtil.deserialize( storedValue, CacheValueWrapper.class );
+                    final String hash = valueWrapper.getCacheKey().getStorageValue();
+                    final int chars = valueWrapper.getPayload().length();
+                    final Instant storeDate = valueWrapper.getExpirationDate();
+                    final String age = Duration.between( storeDate, Instant.now() ).toString();
+                    final CacheDebugItem cacheDebugItem = new CacheDebugItem( hash, age, chars );
+                    items.add( cacheDebugItem );
+                }
+            }
+        }
+        catch ( LocalDBException e )
+        {
+            LOGGER.error( "unexpected error reading debug items: " + e.getMessage(), e );
+        }
+        return Collections.unmodifiableList( items );
+    }
 }

server/src/main/java/password/pwm/svc/cache/MemoryCacheStore.java

@@ -25,11 +25,18 @@ package password.pwm.svc.cache;
 import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import password.pwm.error.PwmUnrecoverableException;
+import password.pwm.util.logging.PwmLogger;
 
+import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
 
 class MemoryCacheStore implements CacheStore
 {
+    private static final PwmLogger LOGGER = PwmLogger.forClass( MemoryCacheStore.class );
     private final Cache<String, CacheValueWrapper> memoryStore;
     private final CacheStoreInfo cacheStoreInfo = new CacheStoreInfo();
 
@@ -81,4 +88,22 @@ class MemoryCacheStore implements CacheStore
     {
         return ( int ) memoryStore.estimatedSize();
     }
+
+    @Override
+    public List<CacheDebugItem> getCacheDebugItems( )
+    {
+        final List<CacheDebugItem> items = new ArrayList<>();
+        final Iterator<CacheValueWrapper> iter = memoryStore.asMap().values().iterator();
+        while ( iter.hasNext() )
+        {
+            final CacheValueWrapper valueWrapper = iter.next();
+            final String hash = valueWrapper.getCacheKey().getStorageValue();
+            final int chars = valueWrapper.getPayload().length();
+            final Instant storeDate = valueWrapper.getExpirationDate();
+            final String age = Duration.between( storeDate, Instant.now() ).toString();
+            final CacheDebugItem cacheDebugItem = new CacheDebugItem( hash, age, chars );
+            items.add( cacheDebugItem );
+        }
+        return Collections.unmodifiableList( items );
+    }
 }

server/src/main/java/password/pwm/svc/email/EmailServerUtil.java

@@ -23,6 +23,7 @@
 
 package password.pwm.svc.email;
 
+import com.sun.mail.smtp.SMTPSendFailedException;
 import password.pwm.AppProperty;
 import password.pwm.PwmConstants;
 import password.pwm.bean.EmailItemBean;
@@ -33,6 +34,7 @@ import password.pwm.error.PwmError;
 import password.pwm.error.PwmUnrecoverableException;
 import password.pwm.http.HttpContentType;
 import password.pwm.util.PasswordData;
+import password.pwm.util.java.JavaHelper;
 import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
@@ -52,7 +54,9 @@ import java.util.Collection;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
+import java.util.Set;
 
 public class EmailServerUtil
 {
@@ -177,23 +181,37 @@ public class EmailServerUtil
         return expandedEmailItem;
     }
 
-    static boolean sendIsRetryable( final Exception e )
+    static boolean examineSendFailure( final Exception e, final Set<Integer> retyableStatusCodes )
     {
         if ( e != null )
         {
-            final Throwable cause = e.getCause();
-            if ( cause instanceof IOException )
             {
-                LOGGER.trace( "message send failure cause is due to an IOException: " + e.getMessage() );
-                return true;
+                final Optional<IOException> optionalIoException = JavaHelper.extractNestedExceptionType( e, IOException.class );
+                if ( optionalIoException.isPresent() )
+                {
+                    LOGGER.trace( "message send failure cause is due to an I/O error: " + optionalIoException.get().getMessage() );
+                    return true;
+                }
             }
-            if ( e instanceof PwmUnrecoverableException )
+
             {
-                if ( ( ( PwmUnrecoverableException ) e ).getError() == PwmError.ERROR_SERVICE_UNREACHABLE )
+                final Optional<SMTPSendFailedException> optionalSmtpSendFailedException = JavaHelper.extractNestedExceptionType( e, SMTPSendFailedException.class );
+                if ( optionalSmtpSendFailedException.isPresent() )
                 {
-                    return true;
+                    final SMTPSendFailedException smtpSendFailedException = optionalSmtpSendFailedException.get();
+                    final int returnCode = smtpSendFailedException.getReturnCode();
+                    LOGGER.trace( "message send failure cause is due to server response code: " + returnCode );
+                    if ( retyableStatusCodes.contains( returnCode ) )
+                    {
+                        return true;
+                    }
                 }
             }
+
+            if ( e instanceof PwmUnrecoverableException )
+            {
+                return ( ( PwmUnrecoverableException ) e ).getError() == PwmError.ERROR_SERVICE_UNREACHABLE;
+            }
         }
         return false;
     }
@@ -286,4 +304,5 @@ public class EmailServerUtil
 
         return transport;
     }
+
 }

server/src/main/java/password/pwm/svc/email/EmailService.java

@@ -26,6 +26,7 @@ import password.pwm.AppProperty;
 import password.pwm.PwmApplication;
 import password.pwm.PwmApplicationMode;
 import password.pwm.bean.EmailItemBean;
+import password.pwm.config.Configuration;
 import password.pwm.config.PwmSetting;
 import password.pwm.config.option.DataStorageMethod;
 import password.pwm.error.ErrorInformation;
@@ -56,10 +57,12 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -77,11 +80,19 @@ public class EmailService implements PwmService
     private final List<EmailServer> servers = new ArrayList<>( );
     private WorkQueueProcessor<EmailItemBean> workQueueProcessor;
     private AtomicLoopIntIncrementer serverIncrementer;
+    private Set<Integer> retryableStatusResponses = Collections.emptySet();
 
     private PwmService.STATUS status = STATUS.NEW;
 
     private final ThreadLocal<EmailConnection> threadLocalTransport = new ThreadLocal<>();
 
+    enum SendFailureMode
+    {
+        RESEND,
+        REQUEUE,
+        DISCARD,
+    }
+
     public void init( final PwmApplication pwmApplication )
             throws PwmException
     {
@@ -113,6 +124,10 @@ public class EmailService implements PwmService
         final LocalDBStoredQueue localDBStoredQueue = LocalDBStoredQueue.createLocalDBStoredQueue( pwmApplication, pwmApplication.getLocalDB(), LocalDB.DB.EMAIL_QUEUE );
 
         workQueueProcessor = new WorkQueueProcessor<>( pwmApplication, localDBStoredQueue, settings, new EmailItemProcessor(), this.getClass() );
+
+        retryableStatusResponses = readRetryableStatusCodes( pwmApplication.getConfig() );
+
+
         status = STATUS.OPEN;
     }
 
@@ -409,7 +424,7 @@ public class EmailService implements PwmService
             }
             LOGGER.error( errorInformation );
 
-            if ( EmailServerUtil.sendIsRetryable( e ) )
+            if ( EmailServerUtil.examineSendFailure( e, retryableStatusResponses ) )
             {
                 LOGGER.error( "error sending email (" + e.getMessage() + ") " + emailItemBean.toDebugString() + ", will retry" );
                 StatisticsManager.incrementStat( pwmApplication, Statistic.EMAIL_SEND_FAILURES );
@@ -457,5 +472,21 @@ public class EmailService implements PwmService
         throw PwmUnrecoverableException.newException( PwmError.ERROR_SERVICE_UNREACHABLE, "unable to reach any configured email server" );
     }
 
+    private static Set<Integer> readRetryableStatusCodes( final Configuration configuration )
+    {
+        final String rawAppProp = configuration.readAppProperty( AppProperty.SMTP_RETRYABLE_SEND_RESPONSE_STATUSES );
+        if ( StringUtil.isEmpty( rawAppProp ) )
+        {
+            return Collections.emptySet();
+        }
+
+        final Set<Integer> returnData = new HashSet<>();
+        for ( final String loopString : rawAppProp.split( "," ) )
+        {
+            final Integer loopInt = Integer.parseInt( loopString );
+            returnData.add( loopInt );
+        }
+        return Collections.unmodifiableSet( returnData );
+    }
 }
 

server/src/main/java/password/pwm/svc/telemetry/HttpTelemetrySender.java

@@ -67,7 +67,7 @@ public class HttpTelemetrySender implements TelemetrySender
         final PwmHttpClient pwmHttpClient = new PwmHttpClient( pwmApplication, SessionLabel.TELEMETRY_SESSION_LABEL, pwmHttpClientConfiguration );
         final String body = JsonUtil.serialize( statsPublishBean );
         final Map<String, String> headers = new HashMap<>();
-        headers.put( HttpHeader.Content_Type.getHttpName(), HttpContentType.json.getHeaderValue() );
+        headers.put( HttpHeader.ContentType.getHttpName(), HttpContentType.json.getHeaderValue() );
         headers.put( HttpHeader.Accept.getHttpName(), PwmConstants.AcceptValue.json.getHeaderValue() );
         final PwmHttpClientRequest pwmHttpClientRequest = new PwmHttpClientRequest(
                 HttpMethod.POST,

server/src/main/java/password/pwm/svc/wordlist/AbstractWordlist.java

@@ -78,7 +78,6 @@ abstract class AbstractWordlist implements Wordlist, PwmService
     protected PwmLogger logger = PwmLogger.forClass( AbstractWordlist.class );
     protected String debugLabel = "Generic Word List";
 
-    protected int storedSize = 0;
     protected boolean debugTrace;
 
     private ErrorInformation lastError;
@@ -173,8 +172,6 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             return;
         }
 
-        //read stored size
-        storedSize = readMetadata().getSize();
         wlStatus = STATUS.OPEN;
     }
 
@@ -259,7 +256,7 @@ abstract class AbstractWordlist implements Wordlist, PwmService
             return 0;
         }
 
-        return storedSize;
+        return readMetadata().getSize();
     }
 
     public synchronized void close( )

server/src/main/java/password/pwm/util/java/JavaHelper.java

@@ -57,6 +57,7 @@ import java.util.GregorianCalendar;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Properties;
 import java.util.TimeZone;
 import java.util.TreeSet;
@@ -594,4 +595,30 @@ public class JavaHelper
         }
         return returnValue;
     }
+
+    public static <T> Optional<T> extractNestedExceptionType( final Exception inputException, final Class<T> exceptionType )
+    {
+        if ( inputException == null )
+        {
+            return Optional.empty();
+        }
+
+        if ( inputException.getClass().isInstance( exceptionType ) )
+        {
+            return Optional.of( ( T ) inputException );
+        }
+
+        Throwable nextException = inputException.getCause();
+        while ( nextException != null )
+        {
+            if ( nextException.getClass().isInstance( exceptionType ) )
+            {
+                return Optional.of( ( T ) inputException );
+            }
+
+            nextException = nextException.getCause();
+        }
+
+        return Optional.empty();
+    }
 }

server/src/main/java/password/pwm/util/java/TimeDuration.java

@@ -32,7 +32,9 @@ import javax.annotation.meta.When;
 import java.io.Serializable;
 import java.math.BigDecimal;
 import java.text.DecimalFormat;
+import java.time.Duration;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
@@ -555,6 +557,15 @@ public class TimeDuration implements Comparable, Serializable
         return pause( this.getTotalMilliseconds() );
     }
 
+    public Duration asDuration()
+    {
+        return Duration.of( this.ms, ChronoUnit.MILLIS );
+    }
+
+    public static TimeDuration fromDuration( final Duration duration )
+    {
+        return new TimeDuration( duration.get( ChronoUnit.MILLIS ) );
+    }
 
     private static class TimeDetail implements Serializable
     {

server/src/main/java/password/pwm/util/operations/ActionExecutor.java

@@ -46,6 +46,7 @@ import password.pwm.util.java.StringUtil;
 import password.pwm.util.logging.PwmLogger;
 import password.pwm.util.macro.MacroMachine;
 
+import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -212,8 +213,15 @@ public class ActionExecutor
             }
             final PwmHttpClientResponse clientResponse = client.makeRequest( clientRequest );
 
-            if ( clientResponse.getStatusCode() != 200 )
+            final List<Integer> successStatus = webAction.getSuccessStatus() == null
+                    ? Collections.emptyList()
+                    : webAction.getSuccessStatus();
+
+            if ( !successStatus.contains( clientResponse.getStatusCode() ) )
             {
+                LOGGER.trace( "response status code " + clientResponse.getStatusCode() + " is not one of the configured success status codes: "
+                        + StringUtil.collectionToString( successStatus ) );
+
                 throw new PwmOperationalException( new ErrorInformation(
                         PwmError.ERROR_SERVICE_UNREACHABLE,
                         "unexpected HTTP status code while calling external web service: "

server/src/main/java/password/pwm/util/queue/SmsQueueManager.java

@@ -591,7 +591,7 @@ public class SmsQueueManager implements PwmService
 
                 if ( !StringUtil.isEmpty( contentType ) && httpMethod == HttpMethod.POST )
                 {
-                    headers.put( HttpHeader.Content_Type.getHttpName(), contentType );
+                    headers.put( HttpHeader.ContentType.getHttpName(), contentType );
                 }
 
                 if ( extraHeaders != null )

server/src/main/java/password/pwm/ws/client/rest/form/RestFormDataClient.java

@@ -83,10 +83,10 @@ public class RestFormDataClient
     {
         final Map<String, String> httpHeaders = new LinkedHashMap<>();
         httpHeaders.put( HttpHeader.Accept.getHttpName(), PwmConstants.AcceptValue.json.getHeaderValue() );
-        httpHeaders.put( HttpHeader.Content_Type.getHttpName(), HttpContentType.json.getHeaderValue() );
+        httpHeaders.put( HttpHeader.ContentType.getHttpName(), HttpContentType.json.getHeaderValue() );
         if ( locale != null )
         {
-            httpHeaders.put( HttpHeader.Accept_Language.getHttpName(), locale.toString() );
+            httpHeaders.put( HttpHeader.AcceptLanguage.getHttpName(), locale.toString() );
         }
 
         {

server/src/main/java/password/pwm/ws/server/RestRequest.java

@@ -81,7 +81,7 @@ public class RestRequest extends PwmHttpRequestWrapper
 
     public HttpContentType readContentType( )
     {
-        return HttpContentType.fromContentTypeHeader( readHeaderValueAsString( HttpHeader.Content_Type ), null );
+        return HttpContentType.fromContentTypeHeader( readHeaderValueAsString( HttpHeader.ContentType ), null );
     }
 
     public HttpContentType readAcceptType( )

server/src/main/java/password/pwm/ws/server/RestServlet.java

@@ -274,11 +274,11 @@ public abstract class RestServlet extends HttpServlet
         }
         else if ( reqContent == null && !anyMatch.isContentMatch() )
         {
-            errorMsg = HttpHeader.Content_Type.getHttpName() + " header is required";
+            errorMsg = HttpHeader.ContentType.getHttpName() + " header is required";
         }
         else if ( !anyMatch.isContentMatch() )
         {
-            errorMsg = HttpHeader.Content_Type.getHttpName() + " header value does not match an available processor";
+            errorMsg = HttpHeader.ContentType.getHttpName() + " header value does not match an available processor";
         }
         else
         {
@@ -318,7 +318,7 @@ public abstract class RestServlet extends HttpServlet
         {
             if ( restRequest.readContentType() == null )
             {
-                final String message = restRequest.getMethod() + " method requires " + HttpHeader.Content_Type.getHttpName() + " header";
+                final String message = restRequest.getMethod() + " method requires " + HttpHeader.ContentType.getHttpName() + " header";
                 throw PwmUnrecoverableException.newException( PwmError.ERROR_UNAUTHORIZED, message );
             }
         }
@@ -342,7 +342,7 @@ public abstract class RestServlet extends HttpServlet
             {
                 case json:
                 {
-                    resp.setHeader( HttpHeader.Content_Type.getHttpName(), HttpContentType.json.getHeaderValue() );
+                    resp.setHeader( HttpHeader.ContentType.getHttpName(), HttpContentType.json.getHeaderValue() );
                     try ( PrintWriter pw = resp.getWriter() )
                     {
                         pw.write( restResultBean.toJson() );
@@ -352,7 +352,7 @@ public abstract class RestServlet extends HttpServlet
 
                 case plain:
                 {
-                    resp.setHeader( HttpHeader.Content_Type.getHttpName(), HttpContentType.plain.getHeaderValue() );
+                    resp.setHeader( HttpHeader.ContentType.getHttpName(), HttpContentType.plain.getHeaderValue() );
                     if ( restResultBean.isError() )
                     {
                         resp.sendError( HttpServletResponse.SC_INTERNAL_SERVER_ERROR, restResultBean.getErrorMessage() );
@@ -396,7 +396,7 @@ public abstract class RestServlet extends HttpServlet
     private static void outputLastHopeError( final String msg, final HttpServletResponse response ) throws IOException
     {
         response.setStatus( HttpServletResponse.SC_INTERNAL_SERVER_ERROR );
-        response.setHeader( HttpHeader.Content_Type.getHttpName(), HttpContentType.json.getHeaderValue() );
+        response.setHeader( HttpHeader.ContentType.getHttpName(), HttpContentType.json.getHeaderValue() );
         try ( PrintWriter pw = response.getWriter() )
         {
             pw.write( "Error: " );

server/src/main/resources/password/pwm/AppProperty.properties

@@ -286,6 +286,7 @@ security.defaultEphemeralHashAlg=SHA512
 security.config.minSecurityKeyLength=32
 seedlist.builtin.path=/WEB-INF/seedlist.zip
 smtp.subjectEncodingCharset=UTF8
+smtp.retryableSendResponseStatus=400,420,421
 telemetry.senderImplementation=password.pwm.svc.telemetry.HttpTelemetrySender
 telemetry.senderSettings={"url":"https://www.pwm-project.org/pwm-data-service/telemetry"}
 telemetry.sendFrequencySeconds=259203

server/src/main/resources/password/pwm/i18n/PwmSetting.properties

@@ -494,7 +494,7 @@ Setting_Description_newUser.redirectUrl=URL to redirect user to after new user r
 Setting_Description_newUser.sms.verification=Enable this option to have @PwmAppName@ send an SMS message to the new user's mobile phone number before it creates the account. The NewUser must verify receipt of the SMS message before @PwmAppName@ creates the account. please insure that the user has entered their SMS information.
 Setting_Description_newUser.token.lifetime=Specify the lifetime a new user email token is valid (in seconds). The default is 0.   When set to 0, the effective value is inherited from the setting <code>@PwmSettingReference\:token.lifetime@</code>
 Setting_Description_newUser.token.lifetime.sms=Specify the lifetime a new user SMS token is valid (in seconds). The default is 0.  When set to 0, the effective value is inherited from the setting <code>@PwmSettingReference\:token.lifetime@</code>
-Setting_Description_newUser.username.definition=<p>Specify the display name, or entry ID that is included in the LDAP naming attribute for the new registered users. Some directories use an LDAP entry instead of a user name.<p>When you enable this setting, the system generates an entryID or an LDAP entry that includes random characters by default.You must specify macros for this setting. For more information about macros, see <a href=https://www.netiq.com/documentation/self-service-password-reset-40/adminguide/data/b19nnbhy.html>Configuring Macros for Messages and Actions</a>.<p>If you leave this field blank, the system does not generate a random user name or entry ID.<p>For example, in the LDAP directory, specify the value as @User:Email@ to display the display name or entry ID for the new registered user as their email address.</p>
+Setting_Description_newUser.username.definition=<p>Specify the display name, or entry ID that is included in the LDAP naming attribute for the new registered users. Some directories use an LDAP entry instead of a user name.<p>When you enable this setting, the system generates an entryID or an LDAP entry that includes random characters by default.You must specify macros for this setting. For more information about macros, see <a href=https://www.netiq.com/documentation/self-service-password-reset-40/adminguide/data/b19nnbhy.html>Configuring Macros for Messages and Actions</a>.<p>If you leave this field blank, the system does not generate a random user name or entry ID.<p>For example, in the LDAP directory, specify the value as @User:Email@ to display the display name or entry ID for the new registered user as their email address.</p><p>When multiple values are entered, if the first value already exists, each value will be tried in order until an unused value is found.</p> 
 Setting_Description_newUser.writeAttributes=Specify the actions the system takes when it creates a user.  The actions will be executed just after the user is created in the LDAP directory.    You can use macros in this setting.
 Setting_Description_notes.noteText=Specify any configuration notes about your system. This option allows you to keep notes about any specific configuration options you have made with the system.
 Setting_Description_oauth.idserver.attributesUrl=Specify the URL of the web service provided by the identity server to return attribute data about the user.

server/src/main/webapp/public/reference/rest.jsp

@@ -94,7 +94,7 @@
             <td class="title" style="font-size: smaller">type</td>
             <td class="title" style="font-size: smaller">description</td>
         </tr>
-        <tr><td>error</td><td>boolean</td><td>true if the operation was successfull</td></tr>
+        <tr><td>error</td><td>boolean</td><td>false if the operation was successfull</td></tr>
         <tr><td>errorCode</td><td>four-digit number</td><td>application error code</td></tr>
         <tr><td>errorMessage</td><td>string</td><td>Localized error message string</td></tr>
         <tr><td>errorDetail</td><td>string</td><td>Error Number, Error ID and debugging detail message if any, English only</td></tr>

server/src/main/webapp/public/resources/js/configeditor-settings-action.js

@@ -34,6 +34,7 @@ ActionHandler.defaultWebValue = {
     headers:{},
     url:"",
     body:"",
+    successStatus:[200],
     username:"",
     password:""
 };
@@ -390,6 +391,9 @@ ActionHandler.addOrEditWebAction = function(keyName, iteration, webActionIter) {
     if (showBody) {
         bodyText += '<tr><td class="key">Body</td><td><textarea style="max-width:400px; height:100px; max-height:100px" class="configStringInput" disabled id="input-' + inputID + '-body' + '"/></textarea></td></tr>';
     }
+    bodyText += '<td class="key">Success Status Codes</td><td><input type="text" class="configstringinput" style="width:300px" disabled id="input-' + inputID + '-successStatus' + '"/>';
+    bodyText += '<button style="margin-left: 15px" id="button-' + inputID + '-successStatus"><span class="btn-icon pwm-icon pwm-icon-list-ul"/> Edit</button>';
+    bodyText += '</td></tr>';
     if (!PWM_MAIN.JSLibrary.isEmpty(value['certificateInfos'])) {
         bodyText += '<tr><td class="key">Certificates</td><td><a id="button-' + inputID + '-certDetail">View Certificates</a></td>';
         bodyText += '</tr>';
@@ -446,6 +450,23 @@ ActionHandler.addOrEditWebAction = function(keyName, iteration, webActionIter) {
                 ActionHandler.write(keyName);
             });
 
+            PWM_MAIN.getObject('input-' + inputID + '-successStatus').value = value['successStatus'] ? value['successStatus'].join() : '';
+            PWM_MAIN.addEventHandler('button-' + inputID + '-successStatus', 'click', function(){
+                var options = {};
+                options['regex'] = '[0-9]{3}';
+                options['title'] = 'Success Status Codes';
+                options['instructions'] = 'Enter the three digit HTTP status codes that will be considered a success if returned by the remote web service.';
+                options['completeFunction'] = function(values){
+                    values.sort();
+                    value['successStatus'] = values;
+                    ActionHandler.write(keyName);
+                    ActionHandler.addOrEditWebAction(keyName, iteration, webActionIter)
+                };
+                var values = 'successStatus' in value ? value['successStatus'] : [];
+                values.sort();
+                options['value'] = values;
+                UILibrary.stringArrayEditorDialog(options);
+            });
 
             if (showBody) {
                 UILibrary.addTextValueToElement('input-' + inputID + '-body', value['body']);

server/src/main/webapp/public/resources/js/configeditor-settings.js

@@ -946,7 +946,7 @@ EmailTableHandler.instrumentRow = function(settingKey, localeName) {
     var editor = function(drawTextArea, type, instructions){
         UILibrary.stringEditorDialog({
             title:'Edit Value - ' + settingData['label'],
-            instructions: instructions,
+            instructions: instructions ? instructions : '',
             textarea:drawTextArea,
             value:PWM_VAR['clientSettingCache'][settingKey][localeName][type],
             completeFunction:function(value){
@@ -2069,13 +2069,15 @@ NamedSecretHandler.init = function(settingKey) {
             });
 
             for (var key in data) {
-                var id = settingKey + '_' + key;
-                PWM_MAIN.addEventHandler('button-deleteRow-' + id,'click',function(){
-                    NamedSecretHandler.deletePassword(settingKey, key);
-                });
-                PWM_MAIN.addEventHandler('button-usage-' + id,'click',function(){
-                    NamedSecretHandler.usagePopup(settingKey, key);
-                });
+                (function (loopKey) {
+                    var id = settingKey + '_' + loopKey;
+                    PWM_MAIN.addEventHandler('button-deleteRow-' + id,'click',function(){
+                        NamedSecretHandler.deletePassword(settingKey, loopKey);
+                    });
+                    PWM_MAIN.addEventHandler('button-usage-' + id,'click',function(){
+                        NamedSecretHandler.usagePopup(settingKey, loopKey);
+                    });
+                })(key);
             }
         });
     }

server/src/main/webapp/public/resources/js/main.js

@@ -281,19 +281,19 @@ PWM_MAIN.applyFormAttributes = function() {
     });
 
     // handle html5 form attribute in JS in case browser (IE) doesn't support it.
-    if(dojo.isIE) {
-        PWM_MAIN.doQuery("button[type=submit][form]", function (element) {
-            /*
-            console.log('added event handler for submit button with form attribute ' + element.id);
-            PWM_MAIN.addEventHandler(element,'click',function(e){
-                PWM_MAIN.stopEvent(e);
-                PWM_VAR['dirtyPageLeaveFlag'] = false;
-                var formID = element.getAttribute('form');
-                PWM_MAIN.handleFormSubmit(PWM_MAIN.getObject(formID));
+    require(["dojo"], function (dojo) {
+        if(dojo.isIE){
+            PWM_MAIN.doQuery("button[type=submit][form]",function(element){
+                console.log('added event handler for submit button with form attribute ' + element.id);
+                PWM_MAIN.addEventHandler(element,'click',function(e){
+                    PWM_MAIN.stopEvent(e);
+                    PWM_VAR['dirtyPageLeaveFlag'] = false;
+                    var formID = element.getAttribute('form');
+                    PWM_MAIN.handleFormSubmit(PWM_MAIN.getObject(formID));
+                });
             });
-            */
-        });
-    }
+        }
+    });
 };
 
 PWM_MAIN.preloadAll = function(nextFunction) {

server/src/main/webapp/public/resources/js/uilibrary.js

@@ -95,6 +95,144 @@ UILibrary.stringEditorDialog = function(options){
     });
 };
 
+UILibrary.stringArrayEditorDialog = function(options){
+    options = options === undefined ? {} : options;
+    var title = 'title' in options ? options['title'] : 'Edit Value';
+    var instructions = 'instructions' in options ? options['instructions'] : null;
+    var completeFunction = 'completeFunction' in options ? options['completeFunction'] : function() {alert('no string array editor dialog complete function')};
+    var regexString = 'regex' in options && options['regex'] ? options['regex'] : '.+';
+    var initialValues = 'value' in options ? options['value'] : [];
+    var placeholder = 'placeholder' in options ? options['placeholder'] : '';
+    var maxvalues = 'maxValues' in options ? options['maxValues'] : 10;
+
+    var regexObject = new RegExp(regexString);
+    var text = '';
+    text += '<div style="visibility: hidden;" id="panel-valueWarning"><span class="pwm-icon pwm-icon-warning message-error"></span>&nbsp;' + PWM_CONFIG.showString('Warning_ValueIncorrectFormat') + '</div>';
+    text += '<br/>';
+
+    if (instructions !== null) {
+        text += '<div style="margin-left: 10px" id="panel-valueInstructions">' + options['instructions'] + '</div>';
+        text += '<br/>';
+    }
+
+    text += '<table class="noborder">';
+    for (var i in initialValues) {
+        text += '<tr class="noborder"><td>';
+        text += '<input style="width: 400px" class="configStringInput" pattern="' + regexString + '" autofocus id="value_' + i + '"/></td>';
+        if (PWM_MAIN.JSLibrary.itemCount(initialValues) > 1) {
+            text += '<td style="width:10px"><span class="delete-row-icon action-icon pwm-icon pwm-icon-times" id="button-value_' + i + '-deleteRow"></span></td>';
+        }
+        text += '</tr>';
+    }
+    text += '</table>';
+
+    if (PWM_MAIN.JSLibrary.itemCount(initialValues) < maxvalues) {
+        text += '<br/>';
+        text += '<button class="btn" id="button-addRow"><span class="btn-icon pwm-icon pwm-icon-plus-square"></span>Add Row</button></td>';
+    }
+
+    var readCurrentValues = function() {
+        var output = [];
+        for (var i in initialValues) {
+            var value = PWM_MAIN.getObject('value_' + i).value;
+            output.push(value);
+        }
+        return output;
+    };
+
+    var inputFunction = function() {
+        PWM_MAIN.getObject('dialog_ok_button').disabled = true;
+        if (PWM_MAIN.JSLibrary.itemCount(initialValues) < maxvalues) {
+            PWM_MAIN.getObject('button-addRow').disabled = true;
+        }
+
+        PWM_MAIN.getObject('panel-valueWarning').style.visibility = 'hidden';
+
+        var passed = true;
+        var allHaveValues = true;
+
+        for (var i in initialValues) {
+            (function(iter) {
+                var value = PWM_MAIN.getObject('value_'+ iter).value;
+                if (value.length > 0) {
+                    var passedRegex = regexObject  !== null && regexObject.test(value);
+                    if (!passedRegex) {
+                        passed = false;
+                    }
+                } else {
+                    allHaveValues = false;
+                }
+            })(i);
+        }
+
+        if (passed && allHaveValues) {
+            PWM_MAIN.getObject('dialog_ok_button').disabled = false;
+            if (PWM_MAIN.JSLibrary.itemCount(initialValues) < maxvalues) {
+                PWM_MAIN.getObject('button-addRow').disabled = false;
+            }
+        } else if (!passed) {
+            PWM_MAIN.getObject('panel-valueWarning').style.visibility = 'visible';
+        }
+
+        PWM_VAR['temp-dialogInputValue'] = readCurrentValues();
+    };
+
+    var okFunction = function() {
+        var value =  PWM_VAR['temp-dialogInputValue'];
+        completeFunction(value);
+    };
+
+    var deleteRow = function(i) {
+        var values = readCurrentValues();
+        values.splice(i,1);
+        options['value'] = values;
+        UILibrary.stringArrayEditorDialog(options);
+    };
+
+    PWM_MAIN.showDialog({
+        title:title,
+        text:text,
+        okAction:okFunction,
+        showCancel:true,
+        showClose: true,
+        allowMove: true,
+        dialogClass: 'auto',
+        loadFunction:function(){
+            for (var i in initialValues) {
+                (function(iter) {
+                    var loopValue = initialValues[iter];
+                    PWM_MAIN.getObject('value_' + i).value = loopValue;
+
+                    if (regexString && regexString.length > 1) {
+                        PWM_MAIN.getObject('value_' + i).setAttribute('pattern',regexString);
+                    }
+                    if (placeholder && placeholder.length > 1) {
+                        PWM_MAIN.getObject('value_' + i).setAttribute('placeholder',placeholder);
+                    }
+                    PWM_MAIN.addEventHandler('value_' + i,'input',function(){
+                        inputFunction();
+                    });
+
+                    PWM_MAIN.addEventHandler('button-value_' + i + '-deleteRow','click',function(){
+                        deleteRow(i);
+                    });
+                })(i);
+            }
+
+            if (PWM_MAIN.JSLibrary.itemCount(initialValues) < maxvalues) {
+                PWM_MAIN.addEventHandler('button-addRow','click',function(){
+                    var values = readCurrentValues();
+                    values.push('');
+                    options['value'] = values;
+                    UILibrary.stringArrayEditorDialog(options);
+                });
+            }
+
+            inputFunction();
+        }
+    });
+};
+
 UILibrary.addTextValueToElement = function(elementID, input) {
     var element = PWM_MAIN.getObject(elementID);
     if (element) {