atlas 的 mysql-bridge.zip

///** // * Licensed to the Apache Software Foundation (ASF) under one // * or more contributor license agreements. See the NOTICE file // * distributed with this work for additional information // * regarding copyright ownership. The ASF licenses this file // * to you under the Apache License, Version 2.0 (the // * "License"); you may not use this file except in compliance // * with the License. You may obtain a copy of the License at // * // * http://www.apache.org/licenses/LICENSE-2.0 // * // * Unless required by applicable law or agreed to in writing, software // * distributed under the License is distributed on an "AS IS" BASIS, // * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // * See the License for the specific language governing permissions and // * limitations under the License. // */ // //package org.apache.atlas.mysql.bridge; // //import com.google.common.annotations.VisibleForTesting; //import com.google.common.hash.Hashing; //import com.sun.jersey.api.client.ClientResponse; //import jline.internal.Log; //import org.apache.atlas.ApplicationProperties; //import org.apache.atlas.AtlasClientV2; //import org.apache.atlas.AtlasServiceException; //import org.apache.atlas.hook.AtlasHookException; //import org.apache.atlas.model.instance.*; //import org.apache.atlas.mysql.model.MysqlDataTypes; //import org.apache.atlas.utils.AuthenticationUtil; //import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; //import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; //import org.apache.commons.cli.ParseException; //import org.apache.commons.collections.CollectionUtils; // //import org.apache.commons.cli.BasicParser; //import org.apache.commons.cli.CommandLine; //import org.apache.commons.cli.Options; //import org.apache.commons.collections.MapUtils; //import org.apache.commons.configuration.Configuration; //import org.apache.commons.lang.StringUtils; //import org.apache.hadoop.security.UserGroupInformation; //import org.slf4j.Logger; //import org.slf4j.LoggerFactory; // //import java.io.BufferedReader; //import java.io.File; //import java.io.FileReader; //import java.io.IOException; //import java.text.SimpleDateFormat; //import java.util.*; // //import static org.apache.atlas.mysql.hook.events.BaseMysqlEvent.*; // ///** // * A Bridge Utility that imports metadata from the Hive Meta Store // * and registers them in Atlas. // */ // //public class MysqlMetaStoreBridgeBak { // private static final Logger LOG = LoggerFactory.getLogger(MysqlMetaStoreBridgeBak.class); // // public static final String CONF_PREFIX = "atlas.hook.mysql."; // public static final String HIVE_CLUSTER_NAME = "atlas.cluster.name"; // public static final String HDFS_PATH_CONVERT_TO_LOWER_CASE = CONF_PREFIX + "hdfs_path.convert_to_lowercase"; // public static final String DEFAULT_CLUSTER_NAME = "mysql"; // public static final String ATLAS_ENDPOINT = "atlas.rest.address"; // public static final String HDFS_PATH = "hdfs_path"; // // private static final int EXIT_CODE_SUCCESS = 0; // private static final int EXIT_CODE_FAILED = 1; // private static final String DEFAULT_ATLAS_URL = "http://localhost:21000/"; // // private String clusterName; // private final AtlasClientV2 atlasClientV2; // private final boolean convertHdfsPathToLowerCase; // // // public static void main(String[] args) { // int exitCode = EXIT_CODE_FAILED; // AtlasClientV2 atlasClientV2 = null; // BufferedReader br = null; // // try { // // Options options = new Options(); // options.addOption("d", "database", true, "Database name"); // options.addOption("t", "table", true, "Table name"); // options.addOption("c", "databaseconfFile", true, "Table name"); // options.addOption("f", "filename", true, "Filename"); // options.addOption("failOnError", false, "failOnError"); // // CommandLine cmd = new BasicParser().parse(options, args); // boolean failOnError = cmd.hasOption("failOnError"); // String databaseToImport = cmd.getOptionValue("d"); // String dbConfFile = cmd.getOptionValue("c"); // String tableToImport = cmd.getOptionValue("t"); // String fileToImport = cmd.getOptionValue("f"); // Configuration atlasConf = ApplicationProperties.get(); // String[] atlasEndpoint = atlasConf.getStringArray(ATLAS_ENDPOINT); // // if (atlasEndpoint == null || atlasEndpoint.length == 0) { // atlasEndpoint = new String[] { DEFAULT_ATLAS_URL }; // } // // Log.info(atlasConf); // // if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) { // // String[] basicAuthUsernamePassword = new String[]{"admin", "admin"} ;//AuthenticationUtil.getBasicAuthenticationInput(); // atlasClientV2 = new AtlasClientV2(atlasEndpoint, basicAuthUsernamePassword); // } else { // // UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); // atlasClientV2 = new AtlasClientV2(ugi, ugi.getShortUserName(), atlasEndpoint); // } // // MysqlMetaStoreBridge mysqlMetaStoreBridge = new MysqlMetaStoreBridge(atlasConf, atlasClientV2); // // HashMap<String,HashMap<String,String>> datasource = new HashMap<>(); // HashMap<String,String> entyMap = new HashMap<>(); // // fileToImport = dbConfFile; // if (StringUtils.isNotEmpty(fileToImport)) { // File f = new File(fileToImport); // System.out.println("klida: " + f.getAbsolutePath()); // if (!f.exists() || !f.canRead()) { // LOG.error("Failed to read the mysql conf file : " + fileToImport); // return; // } // } // // br = new BufferedReader(new FileReader(new File(fileToImport))); // String line = null; // // while((line = br.readLine()) != null) { // if(line.startsWith("#") || line.isEmpty() || line.toUpperCase().contains("_SQOOP_")){ // continue; // } // //System.out.println(line); // datasource = pareConfline(line ,datasource); // } // // mysqlMetaStoreBridge.importMysqlMetadata(datasource); // // exitCode = EXIT_CODE_SUCCESS; // // } catch(ParseException e) { // LOG.error("Failed to parse arguments. Error: ", e.getMessage()); // printUsage(); // } catch(Exception e) { // LOG.error("Import failed", e); // } finally { // if( br !=null) { // try { // br.close(); // } catch (IOException e) { // e.printStackTrace(); // } // } // if( atlasClientV2 !=null) { // atlasClientV2.close(); // } // } // System.exit(exitCode); // // } // // private void importMysqlMetadata(HashMap<String, HashMap<String, String>> datasourceMap) throws Exception { // // clusterName = "mysql"; // for (Map.Entry<String, HashMap<String,String>> entry : datasourceMap.entrySet()) { // System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue()); // AtlasEntityWithExtInfo dbEntity = null; // // try { // dbEntity = importDSandDb("rdbms", entry.getValue().get("DB"), entry.getValue().get("DSNAME"), // entry.getValue().get("IP"), entry.getValue().get("PORT"), entry.getValue().get("USER"), //