项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void replicaInSourcemetastore() throws Exception {
helper.createUnpartitionedTable(toUri(sourceWarehouseUri,DATABASE,SOURCE_UNPARTITIONED_TABLE));
LOG.info(">>>> Table {} ",sourceCatalog.client().getTable(DATABASE,SOURCE_UNPARTITIONED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-same-metastore.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceWarehouseUri,replicaWarehouseUri,housekeepingDbLocation)
.sourcemetastore(sourceCatalog.getThriftConnectionUri(),sourceCatalog.connectionURL(),sourceCatalog.driverClassName())
.replicametastore(sourceCatalog.getThriftConnectionUri()) // Override only this value
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
Table hiveTable = sourceCatalog.client().getTable(DATABASE,TARGET_UNPARTITIONED_TABLE);
assertthat(hiveTable.getdbname(),is(DATABASE));
assertthat(hiveTable.getTableName(),is(TARGET_UNPARTITIONED_TABLE));
assertthat(isExternalTable(hiveTable),is(true));
assertthat(hiveTable.getSd().getCols(),is(DATA_COLUMNS));
}
});
runner.run(config.getAbsolutePath());
}
项目:rug-cli
文件:AbstractCommandTest.java
protected void assertCommandLine(int exitCode,Assertion assertion,boolean includeConf,String... tokens) throws Exception {
String[] commandLine = commandLine(includeConf,tokens);
printCWD();
System.out.println(">>> " + Constants.command()
+ StringUtils.arrayToDelimitedString(commandLine," "));
System.out.println("");
systemOutRule.clearLog();
exit.expectSystemExitWithStatus(exitCode);
exit.checkAssertionAfterwards(assertion);
try {
Main.main(commandLine);
}
catch (Throwable t) {
Chalk.setColorEnabled(true);
throw t;
}
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testNonExistentConfigFile()
throws Exception {
exit.expectSystemExitWithStatus(-1);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log","Could not find config XML file "
+ "'src/main/resources/non_existent_config.xml'." + EOL,systemOut.getLog());
assertEquals("Unexpected system error log","",systemErr.getLog());
}
});
Main.main("-c","src/main/resources/non_existent_config.xml",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testNonExistentClass() throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String expectedExceptionMessage = errorCounterOneMessage.getMessage()
+ EOL;
assertEquals("Unexpected output log",expectedExceptionMessage,systemOut.getLog());
final String cause = "com.puppycrawl.tools.checkstyle.api.CheckstyleException:"
+ " cannot initialize module TreeWalker - ";
assertTrue("Unexpected system error log",systemErr.getLog().startsWith(cause));
}
});
Main.main("-c",getPath("InputMainConfig-non-existent-classname.xml"),getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingTargetFileXmlOutput() throws Exception {
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws IOException {
final String expectedpath = getFilePath("InputMain.java");
final ResourceBundle compilationProperties =
ResourceBundle.getBundle("checkstylecompilation",Locale.ROOT);
final String version = compilationProperties
.getString("checkstyle.compile.version");
assertEquals("Unexpected output log","<?xml version=\"1.0\" encoding=\"UTF-8\"?>" + EOL
+ "<checkstyle version=\"" + version + "\">" + EOL
+ "<file name=\"" + expectedpath + "\">" + EOL
+ "</file>" + EOL
+ "</checkstyle>" + EOL,getPath("InputMainConfig-classname.xml"),"-f","xml",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistentTargetFilePlainOutputToNonExistentFile()
throws Exception {
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log","plain","-o",temporaryFolder.getRoot() + "/output.txt",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingTargetFilePlainOutputToFile()
throws Exception {
final File file = temporaryFolder.newFile("file.output");
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",file.getCanonicalPath(),getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingTargetFilePlainOutputProperties() throws Exception {
mockStatic(Closeables.class);
donothing().when(Closeables.class);
Closeables.closeQuietly(any(InputStream.class));
//exit.expectSystemExitWithStatus(0);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",auditStartMessage.getMessage() + EOL
+ auditFinishMessage.getMessage() + EOL,getPath("InputMainConfig-classname-prop.xml"),"-p",getPath("InputMainMycheckstyle.properties"),getPath("InputMain.java"));
verifyStatic(times(1));
Closeables.closeQuietly(any(InputStream.class));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingTargetFilePlainOutputNonexistentProperties()
throws Exception {
exit.expectSystemExitWithStatus(-1);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log","Could not find file 'nonexistent.properties'."
+ System7.lineseparator(),"nonexistent.properties",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingIncorrectConfigFile()
throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String output = errorCounterOneMessage.getMessage() + EOL;
assertEquals("Unexpected output log",output,systemOut.getLog());
final String errorOutput = "com.puppycrawl.tools.checkstyle.api."
+ "CheckstyleException: unable to parse configuration stream - ";
assertTrue("Unexpected system error log",systemErr.getLog().startsWith(errorOutput));
}
});
Main.main("-c",getPath("InputMainConfig-Incorrect.xml"),getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingIncorrectChildrenInConfigFile()
throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String output = errorCounterOneMessage.getMessage() + EOL;
assertEquals("Unexpected output log",systemOut.getLog());
final String errorOutput = "com.puppycrawl.tools.checkstyle.api."
+ "CheckstyleException: cannot initialize module RegexpSingleline"
+ " - RegexpSingleline is not allowed as a child in RegexpSingleline";
assertTrue("Unexpected system error log",getPath("InputMainConfig-incorrectChildren.xml"),getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExistingIncorrectChildrenInConfigFile2()
throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String output = errorCounterOneMessage.getMessage() + EOL;
assertEquals("Unexpected output log",systemOut.getLog());
final String errorOutput = "com.puppycrawl.tools.checkstyle.api."
+ "CheckstyleException: cannot initialize module TreeWalker"
+ " - JavadocVariable is not allowed as a child in Javadocmethod";
assertTrue("Unexpected system error log",getPath("InputMainConfig-incorrectChildren2.xml"),getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testFileReferenceDuringException() throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String expectedExceptionMessage = auditStartMessage.getMessage() + EOL
+ errorCounterOneMessage.getMessage() + EOL;
assertEquals("Unexpected output log",systemOut.getLog());
final String exceptionFirstLine = "com.puppycrawl.tools.checkstyle.api."
+ "CheckstyleException: Exception was thrown while processing "
+ new File(getNonCompilablePath("InputMainIncorrectClass.java")).getPath()
+ EOL;
assertTrue("Unexpected system error log",systemErr.getLog().startsWith(exceptionFirstLine));
}
});
// We put xml as source to cause parse exception
Main.main("-c",getNonCompilablePath("InputMainIncorrectClass.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testPrintTreeJavadocoption() throws Exception {
final String expected = new String(Files7.readAllBytes(Paths.get(
getPath("InputMainExpectedInputJavadocComment.txt"))),StandardCharsets.UTF_8)
.replaceAll("\\\\r\\\\n","\\\\n");
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",expected,systemOut.getLog().replaceAll("\\\\r\\\\n","\\\\n"));
assertEquals("Unexpected system error log",systemErr.getLog());
}
});
Main.main("-j",getPath("InputMainJavadocComment.javadoc"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testPrintFullTreeOption() throws Exception {
final String expected = new String(Files7.readAllBytes(Paths.get(
getPath("InputMainExpectedInputAstTreeStringPrinterJavadoc.txt"))),StandardCharsets.UTF_8).replaceAll("\\\\r\\\\n",systemErr.getLog());
}
});
Main.main("-J",getPath("InputMainAstTreeStringPrinterJavadoc.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testConflictingOptionsTvsO() throws Exception {
final File file = temporaryFolder.newFile("file.output");
exit.expectSystemExitWithStatus(-1);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log","Option '-t' cannot be used with other options."
+ System7.lineseparator(),systemErr.getLog());
}
});
Main.main("-o","-t",getPath(""));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testCustomrootModule() throws Exception {
TestRootModuleChecker.reset();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",systemErr.getLog());
assertTrue("Invalid Checker state",TestRootModuleChecker.isProcessed());
}
});
Main.main("-c",getPath("InputMainConfig-custom-root-module.xml"),getPath("InputMain.java"));
assertTrue("RootModule should be destroyed",TestRootModuleChecker.isDestroyed());
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testExecuteIgnoredModule() throws Exception {
exit.expectSystemExitWithStatus(-2);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
final String expectedExceptionMessage = errorCounterOneMessage.getMessage() + EOL;
assertEquals("Unexpected output log",getPath("InputMainConfig-non-existent-classname-ignore.xml"),"-executeIgnoredModules",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testCheckerThreadsNumber() throws Exception {
TestRootModuleChecker.reset();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",systemErr.getLog());
assertTrue("Invalid checker state",TestRootModuleChecker.isProcessed());
final DefaultConfiguration config =
(DefaultConfiguration) TestRootModuleChecker.getConfig();
final ThreadModeSettings multiThreadModeSettings = config.getThreadModeSettings();
assertEquals("Invalid checker thread number",4,multiThreadModeSettings.getCheckerThreadsNumber());
assertEquals("Invalid checker thread number",1,multiThreadModeSettings.getTreeWalkerThreadsNumber());
}
});
Main.main("-C","4","-c",getPath("InputMain.java"));
}
项目:checkstyle-backport-jre6
文件:MainTest.java
@Test
public void testTreeWalkerThreadsNumber() throws Exception {
TestRootModuleChecker.reset();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() {
assertEquals("Unexpected output log",multiThreadModeSettings.getTreeWalkerThreadsNumber());
}
});
Main.main("-W",getPath("InputMain.java"));
}
项目:digidoc4j
文件:DigiDoc4JTest.java
@Test
@Ignore("Bug report at https://www.pivotaltracker.com/story/show/107563624")
public void verifyBDocWithWarning() throws IOException {
exit.expectSystemExitWithStatus(0);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
assertthat(sout.getLog(),containsstring("The signer's certificate is not supported by SSCD!"));
}
});
String[] params = new String[]{"-in","testFiles/invalid-containers/warning.asice","-verify","-warnings"};
copyFile(new File("testFiles/yaml-configurations/digidoc4j_ForBDocWarningTest.yaml"),new File("digidoc4j.yaml"));
DigiDoc4J.main(params);
}
项目:migrations
文件:MigrationHookTest.java
@Test
public void testHooks() throws Exception {
exit.expectSystemExit();
exit.checkAssertionAfterwards(new Assertion() {
public void checkAssertion() {
assertEquals("",out.getLog());
}
});
int worklogCounter = 0;
bootstrap();
up();
assertChangelogIntact();
assertWorklogRowCount(worklogCounter += 3);
pending();
assertWorklogRowCount(++worklogCounter);
down();
assertWorklogRowCount(++worklogCounter);
versionDown();
assertWorklogRowCount(++worklogCounter);
versionUp();
assertWorklogRowCount(++worklogCounter);
out.clearLog();
System.exit(0);
}
项目:japicmp
文件:JApiCmpTest.java
@Test
public void testIgnoreMissingClassesByRegExCouldNotLoad() throws IOException,CannotCompileException {
exit.expectSystemExitWithStatus(1);
exit.checkAssertionAfterwards(new Assertion() {
public void checkAssertion() {
String errLogTrimmed = errLog.getLog().trim();
assertthat(errLogTrimmed,containsstring("E: Could not load 'NotExistingSuperclass'".trim()));
}
});
Classpool cp = new Classpool(true);
CtClass ctClassSuperclass = CtClassBuilder.create().name("NotExistingSuperclass").addToClasspool(cp);
CtConstructorBuilder.create().addToClass(ctClassSuperclass);
CtClass ctClass = CtClassBuilder.create().name("Test").withSuperclass(ctClassSuperclass).addToClasspool(cp);
Path oldpath = Paths.get(System.getProperty("user.dir"),"target",JApiCmpTest.class.getSimpleName() + "_old.jar");
createJarFile(oldpath,ctClass);
Path newPath = Paths.get(System.getProperty("user.dir"),JApiCmpTest.class.getSimpleName() + "_new.jar");
createJarFile(newPath,ctClass);
JApiCmp.main(new String[]{"-n",newPath.toString(),oldpath.toString()});
}
项目:japicmp
文件:JApiCmpTest.java
@Test
public void testIgnoreMissingClassesByRegExMissingAreIgnore() throws IOException,CannotCompileException {
exit.checkAssertionAfterwards(new Assertion() {
public void checkAssertion() {
String outLog = JApiCmpTest.this.outLog.getLog().trim();
assertthat(outLog,containsstring("Comparing".trim()));
assertthat(outLog,containsstring("WARNING: You have ignored certain classes".trim()));
}
});
Classpool cp = new Classpool(true);
CtClass ctClassSuperclass = CtClassBuilder.create().name("NotExistingSuperclass").addToClasspool(cp);
CtConstructorBuilder.create().addToClass(ctClassSuperclass);
CtClass ctClass = CtClassBuilder.create().name("Test").withSuperclass(ctClassSuperclass).addToClasspool(cp);
Path oldpath = Paths.get(System.getProperty("user.dir"),oldpath.toString(),CliParser.IGnorE_MISSING_CLASSES_BY_REGEX,".*Superc.*"});
}
项目:denominator
文件:DenominatorTest.java
@Test
public void testInvalidEnvProxySettings() {
exit.expectSystemExitWithStatus(1);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
assertthat(System.getProperty("http.proxyHost")).isEqualTo("localhost");
assertthat(System.getProperty("http.proxyPort")).isEqualTo("80");
assertthat(System.getProperty("https.proxyHost")).isNull();
assertthat(System.getProperty("https.proxyPort")).isNull();
assertthat(systemErrRule.getLog())
.isEqualToIgnoringCase(
"invalid https proxy configuration: no protocol: 10.0.0.1:8443\n");
}
});
Denominator.DenominatorCommand.setProtocolProxyFromEnv("http","http://localhost");
Denominator.DenominatorCommand.setProtocolProxyFromEnv("https","10.0.0.1:8443");
}
项目:circus-train
文件:CircusTrainTest.java
@Test
public void singleYmlFile() throws Exception {
exit.expectSystemExitWithStatus(0);
File ymlFile = temp.newFile("test-application.yml");
List<String> lines = ImmutableList
.<String> builder()
.add("source-catalog:")
.add(" name: source")
.add(" configuration-properties:")
.add(" " + ConfVars.metastoreURIS.varname + ": " + hive.getThriftConnectionUri())
.add("replica-catalog:")
.add(" name: replica")
.add(" hive-metastore-uris: " + hive.getThriftConnectionUri())
.add("table-replications:")
.add(" -")
.add(" source-table:")
.add(" database-name: " + DATABASE)
.add(" table-name: source_" + TABLE)
.add(" replica-table:")
.add(" table-name: replica_" + TABLE)
.add(" table-location: " + temp.newFolder("replica"))
.build();
Files.asCharSink(ymlFile,UTF_8).writeLines(lines);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
assertTrue(hive.client().tableExists(DATABASE,"replica_" + TABLE));
}
});
CircusTrain.main(new String[] { "--config=" + ymlFile.getAbsolutePath() });
}
项目:circus-train
文件:CircusTrainTest.java
@Test
public void singleYmlFileWithUserExtension() throws Exception {
TestLocomotiveListener.testBean = null;
exit.expectSystemExitWithStatus(0);
File ymlFile = temp.newFile("test-application.yml");
List<String> lines = ImmutableList
.<String> builder()
.add("source-catalog:")
.add(" name: source")
.add(" configuration-properties:")
.add(" " + ConfVars.metastoreURIS.varname + ": " + hive.getThriftConnectionUri())
.add("replica-catalog:")
.add(" name: replica")
.add(" hive-metastore-uris: " + hive.getThriftConnectionUri())
.add("table-replications:")
.add(" -")
.add(" source-table:")
.add(" database-name: " + DATABASE)
.add(" table-name: source_" + TABLE)
.add(" replica-table:")
.add(" table-name: replica_" + TABLE)
.add(" table-location: " + temp.newFolder("replica"))
.add("extension-packages: com.hotels.test.extension")
.add("testExtensionConfig: foo")
.build();
Files.asCharSink(ymlFile,UTF_8).writeLines(lines);
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
assertthat(TestLocomotiveListener.testBean.getValue(),is("foo"));
}
});
CircusTrain.main(new String[] { "--config=" + ymlFile.getAbsolutePath() });
}
项目:circus-train
文件:CircusTrainTest.java
@Test
public void twoYmlFiles() throws Exception {
exit.expectSystemExitWithStatus(0);
File ymlFile1 = temp.newFile("test-application1.yml");
File ymlFile2 = temp.newFile("test-application2.yml");
List<String> lines = ImmutableList
.<String> builder()
.add("source-catalog:")
.add(" name: source")
.add(" configuration-properties:")
.add(" " + ConfVars.metastoreURIS.varname + ": " + hive.getThriftConnectionUri())
.add("replica-catalog:")
.add(" name: replica")
.add(" hive-metastore-uris: " + hive.getThriftConnectionUri())
.build();
Files.asCharSink(ymlFile1,UTF_8).writeLines(lines);
lines = ImmutableList
.<String> builder()
.add("table-replications:")
.add(" -")
.add(" source-table:")
.add(" database-name: " + DATABASE)
.add(" table-name: source_" + TABLE)
.add(" replica-table:")
.add(" table-name: replica_" + TABLE)
.add(" table-location: " + temp.newFolder("replica"))
.build();
Files.asCharSink(ymlFile2,"replica_" + TABLE));
}
});
CircusTrain.main(new String[] { "--config=" + ymlFile1.getAbsolutePath() + "," + ymlFile2.getAbsolutePath() });
}
项目:circus-train
文件:CircusTrainS3S3IntegrationTest.java
@Test
public void partitionedTableWithnopartitionsMirror() throws Exception {
final URI sourceTableLocation = toUri("s3a://source/",SOURCE_PARTITIONED_TABLE);
TestUtils.createPartitionedTable(sourceCatalog.client(),SOURCE_PARTITIONED_TABLE,sourceTableLocation);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("partitioned-single-table-with-no-partitions-mirror.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.copierOption(S3S3copierOptions.Keys.S3_ENDPOINT_URI.keyName(),s3Proxy.getProxyUrl())
.sourceConfigurationProperty(ENDPOINT,s3Proxy.getProxyUrl())
.replicaConfigurationProperty(ENDPOINT,s3Proxy.getProxyUrl())
.replicaConfigurationProperty(ACCESS_KEY,s3Proxy.getAccessKey())
.replicaConfigurationProperty(SECRET_KEY,s3Proxy.getSecretKey())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
// Assert location
Table hiveTable = replicaCatalog.client().getTable(DATABASE,TARGET_PARTITIONED_TABLE);
assertthat(hiveTable.getSd().getLocation(),is(sourceTableLocation.toString()));
assertthat(hiveTable.getParameters().get(REPLICATION_EVENT.parameterName()),startsWith("ctp-"));
assertthat(hiveTable.getSd().getCols(),is(DATA_COLUMNS));
// Assert partitions
List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE,TARGET_PARTITIONED_TABLE,(short) -1);
assertthat(partitions.size(),is(0));
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsS3IntegrationTest.java
@Test
public void unpartitionedTable() throws Exception {
final URI sourceTableuri = toUri(sourceWarehouseUri,SOURCE_UNPARTITIONED_TABLE);
helper.createUnpartitionedTable(sourceTableuri);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-hdfs-s3-replication.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.copierOption(S3MapReduceCpOptionsParser.S3_ENDPOINT_URI,TARGET_UNPARTITIONED_TABLE);
String eventId = hiveTable.getParameters().get(REPLICATION_EVENT.parameterName());
URI replicaLocation = toUri("s3a://replica/",TARGET_UNPARTITIONED_TABLE + "/" + eventId);
assertthat(hiveTable.getSd().getLocation(),is(replicaLocation.toString()));
// Assert copied files
File dataFile = new File(sourceTableuri.getPath(),PART_00000);
String fileKeyRegex = String.format("%s/%s/ctt-\\d{8}t\\d{6}.\\d{3}z-\\w{8}/%s",TARGET_UNPARTITIONED_TABLE,PART_00000);
List<S3ObjectSummary> replicaFiles = TestUtils.listObjects(s3Client,"replica");
assertthat(replicaFiles.size(),is(1));
for (S3ObjectSummary objectSummary : replicaFiles) {
assertthat(objectSummary.getSize(),is(dataFile.length()));
assertthat(objectSummary.getKey().matches(fileKeyRegex),is(true));
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsS3IntegrationTest.java
@Test
public void partitionedTableWithnopartitions() throws Exception {
final URI sourceTableuri = toUri(sourceWarehouseUri,sourceTableuri);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("partitioned-single-table-with-no-partitions.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,TARGET_PARTITIONED_TABLE);
String eventId = hiveTable.getParameters().get(REPLICATION_EVENT.parameterName());
URI replicaLocation = toUri("s3a://replica/",is(replicaLocation.toString()));
assertthat(eventId,startsWith("ctp-"));
// Assert partitions
List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE,is(0));
// Assert table directory
List<S3ObjectSummary> replicaFiles = TestUtils.listObjects(s3Client,is(0));
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void partitionedTableHousekeepingEnablednoAudit() throws Exception {
helper.createPartitionedTable(toUri(sourceWarehouseUri,SOURCE_PARTITIONED_TABLE));
LOG.info(">>>> Table {} ",SOURCE_PARTITIONED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("partitioned-single-table-no-housekeeping.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
String jdbcUrl = housekeepingDbJdbcUrl();
try (Connection conn = getConnection(jdbcUrl,HOUSEKEEPING_DB_USER,HOUSEKEEPING_DB_PASSWD)) {
List<LegacyReplicaPath> cleanUpPaths = TestUtils.getCleanUpPaths(conn,"SELECT * FROM circus_train.legacy_replica_path");
assertthat(cleanUpPaths.size(),is(0));
try {
getCleanUpPaths(conn,"SELECT * FROM circus_train.legacy_replica_path_aud");
} catch (sqlException e) {
assertthat(e.getMessage().startsWith("Table \"LEGACY_REPLICA_PATH_AUD\" not found;"),is(true));
}
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void partitionedTableHousekeepingEnabledWithAudit() throws Exception {
helper.createPartitionedTable(toUri(sourceWarehouseUri,SOURCE_PARTITIONED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("partitioned-single-table-with-housekeeping.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
// Assert deleted path
String jdbcUrl = housekeepingDbJdbcUrl();
try (Connection conn = DriverManager.getConnection(jdbcUrl,HOUSEKEEPING_DB_PASSWD)) {
List<LegacyReplicaPath> cleanUpPaths = getCleanUpPaths(conn,is(0));
List<LegacyReplicaPath> cleanUpPathsAudit = getCleanUpPaths(conn,"SELECT * FROM circus_train.legacy_replica_path_aud");
assertthat(cleanUpPathsAudit.size(),is(1));
assertthat(cleanUpPathsAudit.get(0).getEventId(),is("event-124"));
assertthat(cleanUpPathsAudit.get(0).getPathEventId(),is("event-123"));
assertthat(cleanUpPathsAudit.get(0).getPath(),is("file:/foo/bar/event-123/deleteme"));
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void unpartitionedTableHousekeepingEnabledWithAudit() throws Exception {
helper.createUnpartitionedTable(toUri(sourceWarehouseUri,SOURCE_UNPARTITIONED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-with-housekeeping.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
// Assert deleted path
String jdbcUrl = housekeepingDbJdbcUrl();
try (Connection conn = DriverManager.getConnection(jdbcUrl,is("file:/foo/bar/event-123/deleteme"));
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void unpartitionedTableHousekeepingEnablednoAudit() throws Exception {
helper.createUnpartitionedTable(toUri(sourceWarehouseUri,SOURCE_UNPARTITIONED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-no-housekeeping.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,is(true));
}
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void unpartitionedTableMetadataMirror() throws Exception {
helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri,SOURCE_MANAGED_UNPARTITIONED_TABLE));
LOG.info(">>>> Table {} ",SOURCE_MANAGED_UNPARTITIONED_TABLE));
// adjusting the sourceTable,mimicking the change we want to update
Table sourceTable = sourceCatalog.client().getTable(DATABASE,SOURCE_MANAGED_UNPARTITIONED_TABLE);
sourceTable.putToParameters("paramToUpdate","updated");
sourceCatalog.client().alter_table(sourceTable.getdbname(),sourceTable.getTableName(),sourceTable);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-mirror.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
Table hiveTable = replicaCatalog.client().getTable(DATABASE,TARGET_UNPARTITIONED_MANAGED_TABLE);
assertthat(hiveTable.getdbname(),is(TARGET_UNPARTITIONED_MANAGED_TABLE));
// MIRRORED table should be set to EXTERNAL
assertthat(isExternalTable(hiveTable),is(true));
assertthat(hiveTable.getParameters().get("paramToUpdate"),is("updated"));
assertthat(hiveTable.getSd().getCols(),is(DATA_COLUMNS));
File sameAsSourceLocation = new File(sourceWarehouseUri,DATABASE + "/" + SOURCE_MANAGED_UNPARTITIONED_TABLE);
assertthat(hiveTable.getSd().getLocation() + "/",is(sameAsSourceLocation.toURI().toString()));
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void partitionedTableurlEncodedpaths() throws Exception {
helper.createTableWithEncodedPartition(toUri(sourceWarehouseUri,SOURCE_ENCODED_TABLE));
LOG.info(">>>> Table {} ",SOURCE_ENCODED_TABLE));
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("partitioned-single-table-no-housekeeping-url-encoded-paths.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,SOURCE_ENCODED_TABLE);
assertthat(hiveTable.getdbname(),is(SOURCE_ENCODED_TABLE));
List<Partition> partitions = replicaCatalog.client().listPartitions(DATABASE,SOURCE_ENCODED_TABLE,(short) 10);
assertthat(partitions.size(),is(1));
assertthat(partitions.get(0).getSd().getLocation(),endsWith("continent=Europe/country=U%25K"));
assertthat(partitions.get(0).getSd().getLocation(),startsWith(replicaWarehouseUri.toURI().toString()));
Path copiedPartition = new Path(partitions.get(0).getSd().getLocation());
assertTrue(FileSystem.get(replicaCatalog.conf()).exists(copiedPartition));
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void partitionedTableHousekeepingEnablednoAuditPartialReplication() throws Exception {
helper.createPartitionedTable(toUri(sourceWarehouseUri,SOURCE_PARTITIONED_TABLE));
exit.expectSystemExitWithStatus(-2);
File config = dataFolder.getFile("partitioned-single-table-no-housekeeping-partial-replication.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,is(true));
}
}
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void unpartitionedTableReplicateAvroSchema() throws Exception {
helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri,SOURCE_MANAGED_UNPARTITIONED_TABLE));
java.nio.file.Path sourceAvroSchemaPath = Paths.get(sourceWarehouseUri.toString() + "/avro-schema-file.test");
Files.createDirectories(sourceAvroSchemaPath);
String avroSchemaBaseUrl = sourceAvroSchemaPath.toString();
Table sourceTable = sourceCatalog.client().getTable(DATABASE,SOURCE_MANAGED_UNPARTITIONED_TABLE);
sourceTable.putToParameters("avro.schema.url",avroSchemaBaseUrl);
sourceCatalog.client().alter_table(sourceTable.getdbname(),sourceTable);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-avro-schema.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,sourceCatalog.driverClassName())
.replicametastore(replicaCatalog.getThriftConnectionUri())
.build();
exit.checkAssertionAfterwards(new Assertion() {
@Override
public void checkAssertion() throws Exception {
Table replicaHiveTable = replicaCatalog.client().getTable(DATABASE,TARGET_UNPARTITIONED_MANAGED_TABLE);
String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database/";
String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
assertthat(transformedAvroUrl,startsWith(expectedReplicaSchemaUrl));
}
});
runner.run(config.getAbsolutePath());
}
项目:circus-train
文件:CircusTrainHdfsHdfsIntegrationTest.java
@Test
public void unpartitionedTableReplicateAvroSchemaOverride() throws Exception {
helper.createManagedUnpartitionedTable(toUri(sourceWarehouseUri,sourceTable);
exit.expectSystemExitWithStatus(0);
File config = dataFolder.getFile("unpartitioned-single-table-avro-schema-override.yml");
CircusTrainRunner runner = CircusTrainRunner
.builder(DATABASE,TARGET_UNPARTITIONED_MANAGED_TABLE);
String expectedReplicaSchemaUrl = replicaWarehouseUri.toURI().toString() + "ct_database-override/";
String transformedAvroUrl = replicaHiveTable.getParameters().get("avro.schema.url");
assertthat(transformedAvroUrl,startsWith(expectedReplicaSchemaUrl));
}
});
runner.run(config.getAbsolutePath());
}
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 [email protected] 举报,一经查实,本站将立刻删除。