001/*
002 * Copyright 2016-2019 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2016-2019 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.transformations;
022
023
024
025import java.io.File;
026import java.io.FileOutputStream;
027import java.io.InputStream;
028import java.io.OutputStream;
029import java.util.ArrayList;
030import java.util.EnumSet;
031import java.util.Iterator;
032import java.util.LinkedHashMap;
033import java.util.List;
034import java.util.Set;
035import java.util.TreeMap;
036import java.util.concurrent.atomic.AtomicLong;
037import java.util.zip.GZIPOutputStream;
038
039import com.unboundid.ldap.sdk.Attribute;
040import com.unboundid.ldap.sdk.ChangeType;
041import com.unboundid.ldap.sdk.DN;
042import com.unboundid.ldap.sdk.Entry;
043import com.unboundid.ldap.sdk.LDAPException;
044import com.unboundid.ldap.sdk.ResultCode;
045import com.unboundid.ldap.sdk.Version;
046import com.unboundid.ldap.sdk.schema.Schema;
047import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
048import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
049import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
050import com.unboundid.ldif.LDIFException;
051import com.unboundid.ldif.LDIFReader;
052import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
053import com.unboundid.ldif.LDIFReaderEntryTranslator;
054import com.unboundid.ldif.LDIFRecord;
055import com.unboundid.util.ByteStringBuffer;
056import com.unboundid.util.CommandLineTool;
057import com.unboundid.util.Debug;
058import com.unboundid.util.ObjectPair;
059import com.unboundid.util.PassphraseEncryptedOutputStream;
060import com.unboundid.util.StaticUtils;
061import com.unboundid.util.ThreadSafety;
062import com.unboundid.util.ThreadSafetyLevel;
063import com.unboundid.util.args.ArgumentException;
064import com.unboundid.util.args.ArgumentParser;
065import com.unboundid.util.args.BooleanArgument;
066import com.unboundid.util.args.DNArgument;
067import com.unboundid.util.args.FileArgument;
068import com.unboundid.util.args.FilterArgument;
069import com.unboundid.util.args.IntegerArgument;
070import com.unboundid.util.args.ScopeArgument;
071import com.unboundid.util.args.StringArgument;
072
073import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
074
075
076
077/**
078 * This class provides a command-line tool that can be used to apply a number of
079 * transformations to an LDIF file.  The transformations that can be applied
080 * include:
081 * <UL>
082 *   <LI>
083 *     It can scramble the values of a specified set of attributes in a manner
084 *     that attempts to preserve the syntax and consistently scrambles the same
085 *     value to the same representation.
086 *   </LI>
087 *   <LI>
088 *     It can strip a specified set of attributes out of entries.
089 *   </LI>
090 *   <LI>
091 *     It can redact the values of a specified set of attributes, to indicate
092 *     that the values are there but providing no information about what their
093 *     values are.
094 *   </LI>
095 *   <LI>
096 *     It can replace the values of a specified attribute with a given set of
097 *     values.
098 *   </LI>
099 *   <LI>
100 *     It can add an attribute with a given set of values to any entry that does
101 *     not contain that attribute.
102 *   </LI>
103 *   <LI>
104 *     It can replace the values of a specified attribute with a value that
105 *     contains a sequentially-incrementing counter.
106 *   </LI>
107 *   <LI>
108 *     It can strip entries matching a given base DN, scope, and filter out of
109 *     the LDIF file.
110 *   </LI>
111 *   <LI>
112 *     It can perform DN mapping, so that entries that exist below one base DN
113 *     are moved below a different base DN.
114 *   </LI>
115 *   <LI>
116 *     It can perform attribute mapping, to replace uses of one attribute name
117 *     with another.
118 *   </LI>
119 * </UL>
120 */
121@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
122public final class TransformLDIF
123       extends CommandLineTool
124       implements LDIFReaderEntryTranslator
125{
126  /**
127   * The maximum length of any message to write to standard output or standard
128   * error.
129   */
130  private static final int MAX_OUTPUT_LINE_LENGTH =
131       StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
132
133
134
135  // The arguments for use by this program.
136  private BooleanArgument addToExistingValues = null;
137  private BooleanArgument appendToTargetLDIF = null;
138  private BooleanArgument compressTarget = null;
139  private BooleanArgument encryptTarget = null;
140  private BooleanArgument excludeRecordsWithoutChangeType = null;
141  private BooleanArgument excludeNonMatchingEntries = null;
142  private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
143  private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
144  private BooleanArgument hideRedactedValueCount = null;
145  private BooleanArgument processDNs = null;
146  private BooleanArgument sourceCompressed = null;
147  private BooleanArgument sourceContainsChangeRecords = null;
148  private BooleanArgument sourceFromStandardInput = null;
149  private BooleanArgument targetToStandardOutput = null;
150  private DNArgument addAttributeBaseDN = null;
151  private DNArgument excludeEntryBaseDN = null;
152  private DNArgument flattenBaseDN = null;
153  private DNArgument moveSubtreeFrom = null;
154  private DNArgument moveSubtreeTo = null;
155  private FileArgument encryptionPassphraseFile = null;
156  private FileArgument schemaPath = null;
157  private FileArgument sourceLDIF = null;
158  private FileArgument targetLDIF = null;
159  private FilterArgument addAttributeFilter = null;
160  private FilterArgument excludeEntryFilter = null;
161  private FilterArgument flattenExcludeFilter = null;
162  private IntegerArgument initialSequentialValue = null;
163  private IntegerArgument numThreads = null;
164  private IntegerArgument randomSeed = null;
165  private IntegerArgument sequentialValueIncrement = null;
166  private IntegerArgument wrapColumn = null;
167  private ScopeArgument addAttributeScope = null;
168  private ScopeArgument excludeEntryScope = null;
169  private StringArgument addAttributeName = null;
170  private StringArgument addAttributeValue = null;
171  private StringArgument excludeAttribute = null;
172  private StringArgument excludeChangeType  = null;
173  private StringArgument redactAttribute = null;
174  private StringArgument renameAttributeFrom = null;
175  private StringArgument renameAttributeTo = null;
176  private StringArgument replaceValuesAttribute = null;
177  private StringArgument replacementValue = null;
178  private StringArgument scrambleAttribute = null;
179  private StringArgument scrambleJSONField = null;
180  private StringArgument sequentialAttribute = null;
181  private StringArgument textAfterSequentialValue = null;
182  private StringArgument textBeforeSequentialValue = null;
183
184  // A set of thread-local byte stream buffers that will be used to construct
185  // the LDIF representations of records.
186  private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
187       new ThreadLocal<>();
188
189
190
191  /**
192   * Invokes this tool with the provided set of arguments.
193   *
194   * @param  args  The command-line arguments provided to this program.
195   */
196  public static void main(final String... args)
197  {
198    final ResultCode resultCode = main(System.out, System.err, args);
199    if (resultCode != ResultCode.SUCCESS)
200    {
201      System.exit(resultCode.intValue());
202    }
203  }
204
205
206
207  /**
208   * Invokes this tool with the provided set of arguments.
209   *
210   * @param  out   The output stream to use for standard output.  It may be
211   *               {@code null} if standard output should be suppressed.
212   * @param  err   The output stream to use for standard error.  It may be
213   *               {@code null} if standard error should be suppressed.
214   * @param  args  The command-line arguments provided to this program.
215   *
216   * @return  A result code indicating whether processing completed
217   *          successfully.
218   */
219  public static ResultCode main(final OutputStream out, final OutputStream err,
220                                final String... args)
221  {
222    final TransformLDIF tool = new TransformLDIF(out, err);
223    return tool.runTool(args);
224  }
225
226
227
228  /**
229   * Creates a new instance of this tool with the provided information.
230   *
231   * @param  out  The output stream to use for standard output.  It may be
232   *              {@code null} if standard output should be suppressed.
233   * @param  err  The output stream to use for standard error.  It may be
234   *              {@code null} if standard error should be suppressed.
235   */
236  public TransformLDIF(final OutputStream out, final OutputStream err)
237  {
238    super(out, err);
239  }
240
241
242
243  /**
244   * {@inheritDoc}
245   */
246  @Override()
247  public String getToolName()
248  {
249    return "transform-ldif";
250  }
251
252
253
254  /**
255   * {@inheritDoc}
256   */
257  @Override()
258  public String getToolDescription()
259  {
260    return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
261  }
262
263
264
265  /**
266   * {@inheritDoc}
267   */
268  @Override()
269  public String getToolVersion()
270  {
271    return Version.NUMERIC_VERSION_STRING;
272  }
273
274
275
276  /**
277   * {@inheritDoc}
278   */
279  @Override()
280  public boolean supportsInteractiveMode()
281  {
282    return true;
283  }
284
285
286
287  /**
288   * {@inheritDoc}
289   */
290  @Override()
291  public boolean defaultsToInteractiveMode()
292  {
293    return true;
294  }
295
296
297
298  /**
299   * {@inheritDoc}
300   */
301  @Override()
302  public boolean supportsPropertiesFile()
303  {
304    return true;
305  }
306
307
308
309  /**
310   * {@inheritDoc}
311   */
312  @Override()
313  public void addToolArguments(final ArgumentParser parser)
314         throws ArgumentException
315  {
316    // Add arguments pertaining to the source and target LDIF files.
317    sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
318         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
319         false);
320    sourceLDIF.addLongIdentifier("inputLDIF", true);
321    sourceLDIF.addLongIdentifier("source-ldif", true);
322    sourceLDIF.addLongIdentifier("input-ldif", true);
323    sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
324    parser.addArgument(sourceLDIF);
325
326    sourceFromStandardInput = new BooleanArgument(null,
327         "sourceFromStandardInput", 1,
328         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
329    sourceFromStandardInput.addLongIdentifier("source-from-standard-input",
330         true);
331    sourceFromStandardInput.setArgumentGroupName(
332         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
333    parser.addArgument(sourceFromStandardInput);
334    parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
335    parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
336
337    targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
338         INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
339         false);
340    targetLDIF.addLongIdentifier("outputLDIF", true);
341    targetLDIF.addLongIdentifier("target-ldif", true);
342    targetLDIF.addLongIdentifier("output-ldif", true);
343    targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
344    parser.addArgument(targetLDIF);
345
346    targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
347         1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
348    targetToStandardOutput.addLongIdentifier("target-to-standard-output", true);
349    targetToStandardOutput.setArgumentGroupName(
350         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
351    parser.addArgument(targetToStandardOutput);
352    parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
353
354    sourceContainsChangeRecords = new BooleanArgument(null,
355         "sourceContainsChangeRecords",
356         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
357    sourceContainsChangeRecords.addLongIdentifier(
358         "source-contains-change-records", true);
359    sourceContainsChangeRecords.setArgumentGroupName(
360         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
361    parser.addArgument(sourceContainsChangeRecords);
362
363    appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
364         INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
365    appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true);
366    appendToTargetLDIF.setArgumentGroupName(
367         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
368    parser.addArgument(appendToTargetLDIF);
369    parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
370
371    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
372         INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
373    wrapColumn.addLongIdentifier("wrap-column", true);
374    wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
375    parser.addArgument(wrapColumn);
376
377    sourceCompressed = new BooleanArgument('C', "sourceCompressed",
378         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
379    sourceCompressed.addLongIdentifier("inputCompressed", true);
380    sourceCompressed.addLongIdentifier("source-compressed", true);
381    sourceCompressed.addLongIdentifier("input-compressed", true);
382    sourceCompressed.setArgumentGroupName(
383         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
384    parser.addArgument(sourceCompressed);
385
386    compressTarget = new BooleanArgument('c', "compressTarget",
387         INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
388    compressTarget.addLongIdentifier("compressOutput", true);
389    compressTarget.addLongIdentifier("compress", true);
390    compressTarget.addLongIdentifier("compress-target", true);
391    compressTarget.addLongIdentifier("compress-output", true);
392    compressTarget.setArgumentGroupName(
393         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
394    parser.addArgument(compressTarget);
395
396    encryptTarget = new BooleanArgument(null, "encryptTarget",
397         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get());
398    encryptTarget.addLongIdentifier("encryptOutput", true);
399    encryptTarget.addLongIdentifier("encrypt", true);
400    encryptTarget.addLongIdentifier("encrypt-target", true);
401    encryptTarget.addLongIdentifier("encrypt-output", true);
402    encryptTarget.setArgumentGroupName(
403         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
404    parser.addArgument(encryptTarget);
405
406    encryptionPassphraseFile = new FileArgument(null,
407         "encryptionPassphraseFile", false, 1, null,
408         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true,
409         true, false);
410    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
411    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
412         true);
413    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
414         true);
415    encryptionPassphraseFile.setArgumentGroupName(
416         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
417    parser.addArgument(encryptionPassphraseFile);
418
419
420    // Add arguments pertaining to attribute scrambling.
421    scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
422         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
423         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
424    scrambleAttribute.addLongIdentifier("attributeName", true);
425    scrambleAttribute.addLongIdentifier("scramble-attribute", true);
426    scrambleAttribute.addLongIdentifier("attribute-name", true);
427    scrambleAttribute.setArgumentGroupName(
428         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
429    parser.addArgument(scrambleAttribute);
430
431    scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
432         INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
433         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
434              scrambleAttribute.getIdentifierString()));
435    scrambleJSONField.addLongIdentifier("scramble-json-field", true);
436    scrambleJSONField.setArgumentGroupName(
437         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
438    parser.addArgument(scrambleJSONField);
439    parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
440
441    randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
442         INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
443    randomSeed.addLongIdentifier("random-seed", true);
444    randomSeed.setArgumentGroupName(
445         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
446    parser.addArgument(randomSeed);
447
448
449    // Add arguments pertaining to replacing attribute values with a generated
450    // value using a sequential counter.
451    sequentialAttribute = new StringArgument('S', "sequentialAttribute",
452         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
453         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
454              sourceContainsChangeRecords.getIdentifierString()));
455    sequentialAttribute.addLongIdentifier("sequentialAttributeName", true);
456    sequentialAttribute.addLongIdentifier("sequential-attribute", true);
457    sequentialAttribute.addLongIdentifier("sequential-attribute-name", true);
458    sequentialAttribute.setArgumentGroupName(
459         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
460    parser.addArgument(sequentialAttribute);
461    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
462         sequentialAttribute);
463
464    initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
465         false, 1, null,
466         INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
467              sequentialAttribute.getIdentifierString()));
468    initialSequentialValue.addLongIdentifier("initial-sequential-value", true);
469    initialSequentialValue.setArgumentGroupName(
470         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
471    parser.addArgument(initialSequentialValue);
472    parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
473
474    sequentialValueIncrement = new IntegerArgument(null,
475         "sequentialValueIncrement", false, 1, null,
476         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
477              sequentialAttribute.getIdentifierString()));
478    sequentialValueIncrement.addLongIdentifier("sequential-value-increment",
479         true);
480    sequentialValueIncrement.setArgumentGroupName(
481         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
482    parser.addArgument(sequentialValueIncrement);
483    parser.addDependentArgumentSet(sequentialValueIncrement,
484         sequentialAttribute);
485
486    textBeforeSequentialValue = new StringArgument(null,
487         "textBeforeSequentialValue", false, 1, null,
488         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
489              sequentialAttribute.getIdentifierString()));
490    textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value",
491         true);
492    textBeforeSequentialValue.setArgumentGroupName(
493         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
494    parser.addArgument(textBeforeSequentialValue);
495    parser.addDependentArgumentSet(textBeforeSequentialValue,
496         sequentialAttribute);
497
498    textAfterSequentialValue = new StringArgument(null,
499         "textAfterSequentialValue", false, 1, null,
500         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
501              sequentialAttribute.getIdentifierString()));
502    textAfterSequentialValue.addLongIdentifier("text-after-sequential-value",
503         true);
504    textAfterSequentialValue.setArgumentGroupName(
505         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
506    parser.addArgument(textAfterSequentialValue);
507    parser.addDependentArgumentSet(textAfterSequentialValue,
508         sequentialAttribute);
509
510
511    // Add arguments pertaining to attribute value replacement.
512    replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
513         false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
514         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
515              sourceContainsChangeRecords.getIdentifierString()));
516    replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true);
517    replaceValuesAttribute.setArgumentGroupName(
518         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
519    parser.addArgument(replaceValuesAttribute);
520    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
521         replaceValuesAttribute);
522
523    replacementValue = new StringArgument(null, "replacementValue", false, 0,
524         null,
525         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
526              replaceValuesAttribute.getIdentifierString()));
527    replacementValue.addLongIdentifier("replacement-value", true);
528    replacementValue.setArgumentGroupName(
529         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
530    parser.addArgument(replacementValue);
531    parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
532    parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
533
534
535    // Add arguments pertaining to adding missing attributes.
536    addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
537         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
538         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
539              "--addAttributeValue",
540              sourceContainsChangeRecords.getIdentifierString()));
541    addAttributeName.addLongIdentifier("add-attribute-name", true);
542    addAttributeName.setArgumentGroupName(
543         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
544    parser.addArgument(addAttributeName);
545    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
546         addAttributeName);
547
548    addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
549         null,
550         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
551              addAttributeName.getIdentifierString()));
552    addAttributeValue.addLongIdentifier("add-attribute-value", true);
553    addAttributeValue.setArgumentGroupName(
554         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
555    parser.addArgument(addAttributeValue);
556    parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
557    parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
558
559    addToExistingValues = new BooleanArgument(null, "addToExistingValues",
560         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
561              addAttributeName.getIdentifierString(),
562              addAttributeValue.getIdentifierString()));
563    addToExistingValues.addLongIdentifier("add-to-existing-values", true);
564    addToExistingValues.setArgumentGroupName(
565         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
566    parser.addArgument(addToExistingValues);
567    parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
568
569    addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
570         null,
571         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
572              addAttributeName.getIdentifierString()));
573    addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true);
574    addAttributeBaseDN.setArgumentGroupName(
575         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
576    parser.addArgument(addAttributeBaseDN);
577    parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
578
579    addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
580         null,
581         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
582              addAttributeBaseDN.getIdentifierString(),
583              addAttributeName.getIdentifierString()));
584    addAttributeScope.addLongIdentifier("add-attribute-scope", true);
585    addAttributeScope.setArgumentGroupName(
586         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
587    parser.addArgument(addAttributeScope);
588    parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
589
590    addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
591         1, null,
592         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
593              addAttributeName.getIdentifierString()));
594    addAttributeFilter.addLongIdentifier("add-attribute-filter", true);
595    addAttributeFilter.setArgumentGroupName(
596         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
597    parser.addArgument(addAttributeFilter);
598    parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
599
600
601    // Add arguments pertaining to renaming attributes.
602    renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
603         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
604         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
605    renameAttributeFrom.addLongIdentifier("rename-attribute-from", true);
606    renameAttributeFrom.setArgumentGroupName(
607         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
608    parser.addArgument(renameAttributeFrom);
609
610    renameAttributeTo = new StringArgument(null, "renameAttributeTo",
611         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
612         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
613              renameAttributeFrom.getIdentifierString()));
614    renameAttributeTo.addLongIdentifier("rename-attribute-to", true);
615    renameAttributeTo.setArgumentGroupName(
616         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
617    parser.addArgument(renameAttributeTo);
618    parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
619    parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
620
621
622    // Add arguments pertaining to flattening subtrees.
623    flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
624         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
625    flattenBaseDN.addLongIdentifier("flatten-base-dn", true);
626    flattenBaseDN.setArgumentGroupName(
627         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
628    parser.addArgument(flattenBaseDN);
629    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
630         flattenBaseDN);
631
632    flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
633         "flattenAddOmittedRDNAttributesToEntry", 1,
634         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
635    flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
636         "flatten-add-omitted-rdn-attributes-to-entry", true);
637    flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
638         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
639    parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
640    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
641         flattenBaseDN);
642
643    flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
644         "flattenAddOmittedRDNAttributesToRDN", 1,
645         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
646    flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
647         "flatten-add-omitted-rdn-attributes-to-rdn", true);
648    flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
649         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
650    parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
651    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
652         flattenBaseDN);
653
654    flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
655         false, 1, null,
656         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
657    flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true);
658    flattenExcludeFilter.setArgumentGroupName(
659         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
660    parser.addArgument(flattenExcludeFilter);
661    parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
662
663
664    // Add arguments pertaining to moving subtrees.
665    moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
666         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
667    moveSubtreeFrom.addLongIdentifier("move-subtree-from", true);
668    moveSubtreeFrom.setArgumentGroupName(
669         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
670    parser.addArgument(moveSubtreeFrom);
671
672    moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
673         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
674              moveSubtreeFrom.getIdentifierString()));
675    moveSubtreeTo.addLongIdentifier("move-subtree-to", true);
676    moveSubtreeTo.setArgumentGroupName(
677         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
678    parser.addArgument(moveSubtreeTo);
679    parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
680    parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
681
682
683    // Add arguments pertaining to redacting attribute values.
684    redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
685         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
686         INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
687    redactAttribute.addLongIdentifier("redact-attribute", true);
688    redactAttribute.setArgumentGroupName(
689         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
690    parser.addArgument(redactAttribute);
691
692    hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
693         INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
694    hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count",
695         true);
696    hideRedactedValueCount.setArgumentGroupName(
697         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
698    parser.addArgument(hideRedactedValueCount);
699    parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
700
701
702    // Add arguments pertaining to excluding attributes and entries.
703    excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
704         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
705         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
706    excludeAttribute.addLongIdentifier("suppressAttribute", true);
707    excludeAttribute.addLongIdentifier("exclude-attribute", true);
708    excludeAttribute.addLongIdentifier("suppress-attribute", true);
709    excludeAttribute.setArgumentGroupName(
710         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
711    parser.addArgument(excludeAttribute);
712
713    excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
714         null,
715         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
716              sourceContainsChangeRecords.getIdentifierString()));
717    excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true);
718    excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true);
719    excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true);
720    excludeEntryBaseDN.setArgumentGroupName(
721         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
722    parser.addArgument(excludeEntryBaseDN);
723    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
724         excludeEntryBaseDN);
725
726    excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
727         null,
728         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
729              sourceContainsChangeRecords.getIdentifierString()));
730    excludeEntryScope.addLongIdentifier("suppressEntryScope", true);
731    excludeEntryScope.addLongIdentifier("exclude-entry-scope", true);
732    excludeEntryScope.addLongIdentifier("suppress-entry-scope", true);
733    excludeEntryScope.setArgumentGroupName(
734         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
735    parser.addArgument(excludeEntryScope);
736    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
737         excludeEntryScope);
738
739    excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
740         1, null,
741         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
742              sourceContainsChangeRecords.getIdentifierString()));
743    excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true);
744    excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true);
745    excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true);
746    excludeEntryFilter.setArgumentGroupName(
747         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
748    parser.addArgument(excludeEntryFilter);
749    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
750         excludeEntryFilter);
751
752    excludeNonMatchingEntries = new BooleanArgument(null,
753         "excludeNonMatchingEntries",
754         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
755    excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries",
756         true);
757    excludeNonMatchingEntries.setArgumentGroupName(
758         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
759    parser.addArgument(excludeNonMatchingEntries);
760    parser.addDependentArgumentSet(excludeNonMatchingEntries,
761         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
762
763
764    // Add arguments for excluding records based on their change types.
765    excludeChangeType = new StringArgument(null, "excludeChangeType",
766         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(),
767         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(),
768         StaticUtils.setOf("add", "delete", "modify", "moddn"));
769    excludeChangeType.addLongIdentifier("exclude-change-type", true);
770    excludeChangeType.addLongIdentifier("exclude-changetype", true);
771    excludeChangeType.setArgumentGroupName(
772         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
773    parser.addArgument(excludeChangeType);
774
775
776    // Add arguments for excluding records that don't have a change type.
777    excludeRecordsWithoutChangeType = new BooleanArgument(null,
778         "excludeRecordsWithoutChangeType", 1,
779         INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get());
780    excludeRecordsWithoutChangeType.addLongIdentifier(
781         "exclude-records-without-change-type", true);
782    excludeRecordsWithoutChangeType.addLongIdentifier(
783         "exclude-records-without-changetype", true);
784    excludeRecordsWithoutChangeType.setArgumentGroupName(
785         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
786    parser.addArgument(excludeRecordsWithoutChangeType);
787
788
789    // Add the remaining arguments.
790    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
791         INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
792         true, true, false, false);
793    schemaPath.addLongIdentifier("schemaFile", true);
794    schemaPath.addLongIdentifier("schemaDirectory", true);
795    schemaPath.addLongIdentifier("schema-path", true);
796    schemaPath.addLongIdentifier("schema-file", true);
797    schemaPath.addLongIdentifier("schema-directory", true);
798    parser.addArgument(schemaPath);
799
800    numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
801         INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
802         1);
803    numThreads.addLongIdentifier("num-threads", true);
804    parser.addArgument(numThreads);
805
806    processDNs = new BooleanArgument('d', "processDNs",
807         INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
808    processDNs.addLongIdentifier("process-dns", true);
809    parser.addArgument(processDNs);
810
811
812    // Ensure that at least one kind of transformation was requested.
813    parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
814         replaceValuesAttribute, addAttributeName, renameAttributeFrom,
815         flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
816         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter,
817         excludeChangeType, excludeRecordsWithoutChangeType);
818  }
819
820
821
822  /**
823   * {@inheritDoc}
824   */
825  @Override()
826  public void doExtendedArgumentValidation()
827         throws ArgumentException
828  {
829    // Ideally, exactly one of the targetLDIF and targetToStandardOutput
830    // arguments should always be provided.  But in order to preserve backward
831    // compatibility with a legacy scramble-ldif tool, we will allow both to be
832    // omitted if either --scrambleAttribute or --sequentialArgument is
833    // provided.  In that case, the path of the output file will be the path of
834    // the first input file with ".scrambled" appended to it.
835    if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
836    {
837      if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
838      {
839        throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
840             targetLDIF.getIdentifierString(),
841             targetToStandardOutput.getIdentifierString()));
842      }
843    }
844
845
846    // Make sure that the --renameAttributeFrom and --renameAttributeTo
847    // arguments were provided an equal number of times.
848    final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
849    final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
850    if (renameFromOccurrences != renameToOccurrences)
851    {
852      throw new ArgumentException(
853           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
854                renameAttributeFrom.getIdentifierString(),
855                renameAttributeTo.getIdentifierString()));
856    }
857
858
859    // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
860    // provided an equal number of times.
861    final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
862    final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
863    if (moveFromOccurrences != moveToOccurrences)
864    {
865      throw new ArgumentException(
866           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
867                moveSubtreeFrom.getIdentifierString(),
868                moveSubtreeTo.getIdentifierString()));
869    }
870  }
871
872
873
874  /**
875   * {@inheritDoc}
876   */
877  @Override()
878  public ResultCode doToolProcessing()
879  {
880    final Schema schema;
881    try
882    {
883      schema = getSchema();
884    }
885    catch (final LDAPException le)
886    {
887      wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
888      return le.getResultCode();
889    }
890
891
892    // If an encryption passphrase file is provided, then get the passphrase
893    // from it.
894    String encryptionPassphrase = null;
895    if (encryptionPassphraseFile.isPresent())
896    {
897      try
898      {
899        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
900             encryptionPassphraseFile.getValue());
901      }
902      catch (final LDAPException e)
903      {
904        wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage());
905        return e.getResultCode();
906      }
907    }
908
909
910    // Create the translators to use to apply the transformations.
911    final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
912         new ArrayList<>(10);
913    final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
914         new ArrayList<>(10);
915
916    final AtomicLong excludedEntryCount = new AtomicLong(0L);
917    createTranslators(entryTranslators, changeRecordTranslators,
918         schema, excludedEntryCount);
919
920    final AggregateLDIFReaderEntryTranslator entryTranslator =
921         new AggregateLDIFReaderEntryTranslator(entryTranslators);
922    final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
923         new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
924
925
926    // Determine the path to the target file to be written.
927    final File targetFile;
928    if (targetLDIF.isPresent())
929    {
930      targetFile = targetLDIF.getValue();
931    }
932    else if (targetToStandardOutput.isPresent())
933    {
934      targetFile = null;
935    }
936    else
937    {
938      targetFile =
939           new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
940    }
941
942
943    // Create the LDIF reader.
944    final LDIFReader ldifReader;
945    try
946    {
947      final InputStream inputStream;
948      if (sourceLDIF.isPresent())
949      {
950        final ObjectPair<InputStream,String> p =
951             ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(),
952                  encryptionPassphrase, getOut(), getErr());
953        inputStream = p.getFirst();
954        if ((encryptionPassphrase == null) && (p.getSecond() != null))
955        {
956          encryptionPassphrase = p.getSecond();
957        }
958      }
959      else
960      {
961        inputStream = System.in;
962      }
963
964      ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
965           entryTranslator, changeRecordTranslator);
966      if (schema != null)
967      {
968        ldifReader.setSchema(schema);
969      }
970    }
971    catch (final Exception e)
972    {
973      Debug.debugException(e);
974      wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
975           ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
976                StaticUtils.getExceptionMessage(e)));
977      return ResultCode.LOCAL_ERROR;
978    }
979
980
981    ResultCode resultCode = ResultCode.SUCCESS;
982    OutputStream outputStream = null;
983processingBlock:
984    try
985    {
986      // Create the output stream to use to write the transformed data.
987      try
988      {
989        if (targetFile == null)
990        {
991          outputStream = getOut();
992        }
993        else
994        {
995          outputStream =
996               new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
997        }
998
999        if (encryptTarget.isPresent())
1000        {
1001          if (encryptionPassphrase == null)
1002          {
1003            encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase(
1004                 false, true, getOut(), getErr());
1005          }
1006
1007          outputStream = new PassphraseEncryptedOutputStream(
1008               encryptionPassphrase, outputStream);
1009        }
1010
1011        if (compressTarget.isPresent())
1012        {
1013          outputStream = new GZIPOutputStream(outputStream);
1014        }
1015      }
1016      catch (final Exception e)
1017      {
1018        Debug.debugException(e);
1019        wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1020             ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
1021                  targetFile.getAbsolutePath(),
1022                  StaticUtils.getExceptionMessage(e)));
1023        resultCode = ResultCode.LOCAL_ERROR;
1024        break processingBlock;
1025      }
1026
1027
1028      // Read the source data one record at a time.  The transformations will
1029      // automatically be applied by the LDIF reader's translators, and even if
1030      // there are multiple reader threads, we're guaranteed to get the results
1031      // in the right order.
1032      long entriesWritten = 0L;
1033      while (true)
1034      {
1035        final LDIFRecord ldifRecord;
1036        try
1037        {
1038          ldifRecord = ldifReader.readLDIFRecord();
1039        }
1040        catch (final LDIFException le)
1041        {
1042          Debug.debugException(le);
1043          if (le.mayContinueReading())
1044          {
1045            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1046                 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
1047                      StaticUtils.getExceptionMessage(le)));
1048            if (resultCode == ResultCode.SUCCESS)
1049            {
1050              resultCode = ResultCode.PARAM_ERROR;
1051            }
1052            continue;
1053          }
1054          else
1055          {
1056            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1057                 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
1058                      StaticUtils.getExceptionMessage(le)));
1059            if (resultCode == ResultCode.SUCCESS)
1060            {
1061              resultCode = ResultCode.PARAM_ERROR;
1062            }
1063            break processingBlock;
1064          }
1065        }
1066        catch (final Exception e)
1067        {
1068          Debug.debugException(e);
1069          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1070               ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1071                    StaticUtils.getExceptionMessage(e)));
1072          resultCode = ResultCode.LOCAL_ERROR;
1073          break processingBlock;
1074        }
1075
1076
1077        // If the LDIF record is null, then we've run out of records so we're
1078        // done.
1079        if (ldifRecord == null)
1080        {
1081          break;
1082        }
1083
1084
1085        // Write the record to the output stream.
1086        try
1087        {
1088          if (ldifRecord instanceof PreEncodedLDIFEntry)
1089          {
1090            outputStream.write(
1091                 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1092          }
1093          else
1094          {
1095            final ByteStringBuffer buffer = getBuffer();
1096            if (wrapColumn.isPresent())
1097            {
1098              ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1099            }
1100            else
1101            {
1102              ldifRecord.toLDIF(buffer, 0);
1103            }
1104            buffer.append(StaticUtils.EOL_BYTES);
1105            buffer.write(outputStream);
1106          }
1107        }
1108        catch (final Exception e)
1109        {
1110          Debug.debugException(e);
1111          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1112               ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1113                    StaticUtils.getExceptionMessage(e)));
1114          resultCode = ResultCode.LOCAL_ERROR;
1115          break processingBlock;
1116        }
1117
1118
1119        // If we've written a multiple of 1000 entries, print a progress
1120        // message.
1121        entriesWritten++;
1122        if ((! targetToStandardOutput.isPresent()) &&
1123            ((entriesWritten % 1000L) == 0))
1124        {
1125          final long numExcluded = excludedEntryCount.get();
1126          if (numExcluded > 0L)
1127          {
1128            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1129                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1130                      entriesWritten, numExcluded));
1131          }
1132          else
1133          {
1134            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1135                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1136                      entriesWritten));
1137          }
1138        }
1139      }
1140
1141
1142      if (! targetToStandardOutput.isPresent())
1143      {
1144        final long numExcluded = excludedEntryCount.get();
1145        if (numExcluded > 0L)
1146        {
1147          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1148               INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1149                    numExcluded));
1150        }
1151        else
1152        {
1153          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1154               INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1155        }
1156      }
1157    }
1158    finally
1159    {
1160      if (outputStream != null)
1161      {
1162        try
1163        {
1164          outputStream.close();
1165        }
1166        catch (final Exception e)
1167        {
1168          Debug.debugException(e);
1169          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1170               ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1171                    targetFile.getAbsolutePath(),
1172                    StaticUtils.getExceptionMessage(e)));
1173          if (resultCode == ResultCode.SUCCESS)
1174          {
1175            resultCode = ResultCode.LOCAL_ERROR;
1176          }
1177        }
1178      }
1179
1180      try
1181      {
1182        ldifReader.close();
1183      }
1184      catch (final Exception e)
1185      {
1186        Debug.debugException(e);
1187        // We can ignore this.
1188      }
1189    }
1190
1191
1192    return resultCode;
1193  }
1194
1195
1196
1197  /**
1198   * Retrieves the schema that should be used for processing.
1199   *
1200   * @return  The schema that was created.
1201   *
1202   * @throws  LDAPException  If a problem is encountered while retrieving the
1203   *                         schema.
1204   */
1205  private Schema getSchema()
1206          throws LDAPException
1207  {
1208    // If any schema paths were specified, then load the schema only from those
1209    // paths.
1210    if (schemaPath.isPresent())
1211    {
1212      final ArrayList<File> schemaFiles = new ArrayList<>(10);
1213      for (final File path : schemaPath.getValues())
1214      {
1215        if (path.isFile())
1216        {
1217          schemaFiles.add(path);
1218        }
1219        else
1220        {
1221          final TreeMap<String,File> fileMap = new TreeMap<>();
1222          for (final File schemaDirFile : path.listFiles())
1223          {
1224            final String name = schemaDirFile.getName();
1225            if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1226            {
1227              fileMap.put(name, schemaDirFile);
1228            }
1229          }
1230          schemaFiles.addAll(fileMap.values());
1231        }
1232      }
1233
1234      if (schemaFiles.isEmpty())
1235      {
1236        throw new LDAPException(ResultCode.PARAM_ERROR,
1237             ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1238                  schemaPath.getIdentifierString()));
1239      }
1240      else
1241      {
1242        try
1243        {
1244          return Schema.getSchema(schemaFiles);
1245        }
1246        catch (final Exception e)
1247        {
1248          Debug.debugException(e);
1249          throw new LDAPException(ResultCode.LOCAL_ERROR,
1250               ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1251                    StaticUtils.getExceptionMessage(e)));
1252        }
1253      }
1254    }
1255    else
1256    {
1257      // If the INSTANCE_ROOT environment variable is set and it refers to a
1258      // directory that has a config/schema subdirectory that has one or more
1259      // schema files in it, then read the schema from that directory.
1260      try
1261      {
1262        final String instanceRootStr =
1263             StaticUtils.getEnvironmentVariable("INSTANCE_ROOT");
1264        if (instanceRootStr != null)
1265        {
1266          final File instanceRoot = new File(instanceRootStr);
1267          final File configDir = new File(instanceRoot, "config");
1268          final File schemaDir = new File(configDir, "schema");
1269          if (schemaDir.exists())
1270          {
1271            final TreeMap<String,File> fileMap = new TreeMap<>();
1272            for (final File schemaDirFile : schemaDir.listFiles())
1273            {
1274              final String name = schemaDirFile.getName();
1275              if (schemaDirFile.isFile() &&
1276                  name.toLowerCase().endsWith(".ldif"))
1277              {
1278                fileMap.put(name, schemaDirFile);
1279              }
1280            }
1281
1282            if (! fileMap.isEmpty())
1283            {
1284              return Schema.getSchema(new ArrayList<>(fileMap.values()));
1285            }
1286          }
1287        }
1288      }
1289      catch (final Exception e)
1290      {
1291        Debug.debugException(e);
1292      }
1293    }
1294
1295
1296    // If we've gotten here, then just return null and the tool will try to use
1297    // the default standard schema.
1298    return null;
1299  }
1300
1301
1302
1303  /**
1304   * Creates the entry and change record translators that will be used to
1305   * perform the transformations.
1306   *
1307   * @param  entryTranslators         A list to which all created entry
1308   *                                  translators should be written.
1309   * @param  changeRecordTranslators  A list to which all created change record
1310   *                                  translators should be written.
1311   * @param  schema                   The schema to use when processing.
1312   * @param  excludedEntryCount       A counter used to keep track of the number
1313   *                                  of entries that have been excluded from
1314   *                                  the result set.
1315   */
1316  private void createTranslators(
1317       final List<LDIFReaderEntryTranslator> entryTranslators,
1318       final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1319       final Schema schema, final AtomicLong excludedEntryCount)
1320  {
1321    if (scrambleAttribute.isPresent())
1322    {
1323      final Long seed;
1324      if (randomSeed.isPresent())
1325      {
1326        seed = randomSeed.getValue().longValue();
1327      }
1328      else
1329      {
1330        seed = null;
1331      }
1332
1333      final ScrambleAttributeTransformation t =
1334           new ScrambleAttributeTransformation(schema, seed,
1335                processDNs.isPresent(), scrambleAttribute.getValues(),
1336                scrambleJSONField.getValues());
1337      entryTranslators.add(t);
1338      changeRecordTranslators.add(t);
1339    }
1340
1341    if (sequentialAttribute.isPresent())
1342    {
1343      final long initialValue;
1344      if (initialSequentialValue.isPresent())
1345      {
1346        initialValue = initialSequentialValue.getValue().longValue();
1347      }
1348      else
1349      {
1350        initialValue = 0L;
1351      }
1352
1353      final long incrementAmount;
1354      if (sequentialValueIncrement.isPresent())
1355      {
1356        incrementAmount = sequentialValueIncrement.getValue().longValue();
1357      }
1358      else
1359      {
1360        incrementAmount = 1L;
1361      }
1362
1363      for (final String attrName : sequentialAttribute.getValues())
1364      {
1365
1366
1367        final ReplaceWithCounterTransformation t =
1368             new ReplaceWithCounterTransformation(schema, attrName,
1369                  initialValue, incrementAmount,
1370                  textBeforeSequentialValue.getValue(),
1371                  textAfterSequentialValue.getValue(), processDNs.isPresent());
1372        entryTranslators.add(t);
1373      }
1374    }
1375
1376    if (replaceValuesAttribute.isPresent())
1377    {
1378      final ReplaceAttributeTransformation t =
1379           new ReplaceAttributeTransformation(schema,
1380                replaceValuesAttribute.getValue(),
1381                replacementValue.getValues());
1382      entryTranslators.add(t);
1383    }
1384
1385    if (addAttributeName.isPresent())
1386    {
1387      final AddAttributeTransformation t = new AddAttributeTransformation(
1388           schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1389           addAttributeFilter.getValue(),
1390           new Attribute(addAttributeName.getValue(), schema,
1391                addAttributeValue.getValues()),
1392           (! addToExistingValues.isPresent()));
1393      entryTranslators.add(t);
1394    }
1395
1396    if (renameAttributeFrom.isPresent())
1397    {
1398      final Iterator<String> renameFromIterator =
1399           renameAttributeFrom.getValues().iterator();
1400      final Iterator<String> renameToIterator =
1401           renameAttributeTo.getValues().iterator();
1402      while (renameFromIterator.hasNext())
1403      {
1404        final RenameAttributeTransformation t =
1405             new RenameAttributeTransformation(schema,
1406                  renameFromIterator.next(), renameToIterator.next(),
1407                  processDNs.isPresent());
1408        entryTranslators.add(t);
1409        changeRecordTranslators.add(t);
1410      }
1411    }
1412
1413    if (flattenBaseDN.isPresent())
1414    {
1415      final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1416           schema, flattenBaseDN.getValue(),
1417           flattenAddOmittedRDNAttributesToEntry.isPresent(),
1418           flattenAddOmittedRDNAttributesToRDN.isPresent(),
1419           flattenExcludeFilter.getValue());
1420      entryTranslators.add(t);
1421    }
1422
1423    if (moveSubtreeFrom.isPresent())
1424    {
1425      final Iterator<DN> moveFromIterator =
1426           moveSubtreeFrom.getValues().iterator();
1427      final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1428      while (moveFromIterator.hasNext())
1429      {
1430        final MoveSubtreeTransformation t =
1431             new MoveSubtreeTransformation(moveFromIterator.next(),
1432                  moveToIterator.next());
1433        entryTranslators.add(t);
1434        changeRecordTranslators.add(t);
1435      }
1436    }
1437
1438    if (redactAttribute.isPresent())
1439    {
1440      final RedactAttributeTransformation t = new RedactAttributeTransformation(
1441           schema, processDNs.isPresent(),
1442           (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1443      entryTranslators.add(t);
1444      changeRecordTranslators.add(t);
1445    }
1446
1447    if (excludeAttribute.isPresent())
1448    {
1449      final ExcludeAttributeTransformation t =
1450           new ExcludeAttributeTransformation(schema,
1451                excludeAttribute.getValues());
1452      entryTranslators.add(t);
1453      changeRecordTranslators.add(t);
1454    }
1455
1456    if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1457        excludeEntryFilter.isPresent())
1458    {
1459      final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1460           schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1461           excludeEntryFilter.getValue(),
1462           (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1463      entryTranslators.add(t);
1464    }
1465
1466    if (excludeChangeType.isPresent())
1467    {
1468      final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class);
1469      for (final String changeTypeName : excludeChangeType.getValues())
1470      {
1471        changeTypes.add(ChangeType.forName(changeTypeName));
1472      }
1473
1474      changeRecordTranslators.add(
1475           new ExcludeChangeTypeTransformation(changeTypes));
1476    }
1477
1478    if (excludeRecordsWithoutChangeType.isPresent())
1479    {
1480      entryTranslators.add(new ExcludeAllEntriesTransformation());
1481    }
1482
1483    entryTranslators.add(this);
1484  }
1485
1486
1487
1488  /**
1489   * {@inheritDoc}
1490   */
1491  @Override()
1492  public LinkedHashMap<String[],String> getExampleUsages()
1493  {
1494    final LinkedHashMap<String[],String> examples =
1495         new LinkedHashMap<>(StaticUtils.computeMapCapacity(4));
1496
1497    examples.put(
1498         new String[]
1499         {
1500           "--sourceLDIF", "input.ldif",
1501           "--targetLDIF", "scrambled.ldif",
1502           "--scrambleAttribute", "givenName",
1503           "--scrambleAttribute", "sn",
1504           "--scrambleAttribute", "cn",
1505           "--numThreads", "10",
1506           "--schemaPath", "/ds/config/schema",
1507           "--processDNs"
1508         },
1509         INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1510
1511    examples.put(
1512         new String[]
1513         {
1514           "--sourceLDIF", "input.ldif",
1515           "--targetLDIF", "sequential.ldif",
1516           "--sequentialAttribute", "uid",
1517           "--initialSequentialValue", "1",
1518           "--sequentialValueIncrement", "1",
1519           "--textBeforeSequentialValue", "user.",
1520           "--numThreads", "10",
1521           "--schemaPath", "/ds/config/schema",
1522           "--processDNs"
1523         },
1524         INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1525
1526    examples.put(
1527         new String[]
1528         {
1529           "--sourceLDIF", "input.ldif",
1530           "--targetLDIF", "added-organization.ldif",
1531           "--addAttributeName", "o",
1532           "--addAttributeValue", "Example Corp.",
1533           "--addAttributeFilter", "(objectClass=person)",
1534           "--numThreads", "10",
1535           "--schemaPath", "/ds/config/schema"
1536         },
1537         INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1538
1539    examples.put(
1540         new String[]
1541         {
1542           "--sourceLDIF", "input.ldif",
1543           "--targetLDIF", "rebased.ldif",
1544           "--moveSubtreeFrom", "o=example.com",
1545           "--moveSubtreeTo", "dc=example,dc=com",
1546           "--numThreads", "10",
1547           "--schemaPath", "/ds/config/schema"
1548         },
1549         INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1550
1551    return examples;
1552  }
1553
1554
1555
1556  /**
1557   * {@inheritDoc}
1558   */
1559  @Override()
1560  public Entry translate(final Entry original, final long firstLineNumber)
1561         throws LDIFException
1562  {
1563    final ByteStringBuffer buffer = getBuffer();
1564    if (wrapColumn.isPresent())
1565    {
1566      original.toLDIF(buffer, wrapColumn.getValue());
1567    }
1568    else
1569    {
1570      original.toLDIF(buffer, 0);
1571    }
1572    buffer.append(StaticUtils.EOL_BYTES);
1573
1574    return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1575  }
1576
1577
1578
1579  /**
1580   * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1581   *
1582   * @return  A byte string buffer that can be used to perform LDIF encoding.
1583   */
1584  private ByteStringBuffer getBuffer()
1585  {
1586    ByteStringBuffer buffer = byteStringBuffers.get();
1587    if (buffer == null)
1588    {
1589      buffer = new ByteStringBuffer();
1590      byteStringBuffers.set(buffer);
1591    }
1592    else
1593    {
1594      buffer.clear();
1595    }
1596
1597    return buffer;
1598  }
1599}