- 解压
- 修改sqoop-env-template.sh文件名为sqoop-env.sh;并修改一下内容!!
[root@leader bin]# echo $HADOOP_HOME
/opt/hadoop-3.2.2
[root@leader bin]# echo $HBASE_HOME
/opt/hbase-2.3.6
[root@leader bin]# echo $HIVE_HOME
/opt/hive-3.1.2
---
[root@leader conf]# vim sqoop-env.sh
#export HADOOP_COMMON_HOME=
export HADOOP_COMMON_HOME=/opt/hadoop-3.2.2
#Set path to where hadoop-*-core.jar is available
#export HADOOP_MAPRED_HOME=
export HADOOP_MAPRED_HOME=/opt/hadoop-3.2.2
#set the path to where bin/hbase is available
#export HBASE_HOME=
export HBASE_HOME=/opt/hbase-2.3.6
#Set the path to where bin/hive is available
#export HIVE_HOME=
export HIVE_HOME=/opt/hive-3.1.2
#Set the path for where zookeper config dir is
#export ZOOCFGDIR=
export ZOOCFGDIR=/opt/apache-zookeeper-3.6.3-bin
- 添加mysql驱动包到lib目录下
[root@leader lib]# pwd
/opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar opencsv-2.3.jar
commons-codec-1.4.jar paranamer-2.7.jar
commons-compress-1.8.1.jar parquet-avro-1.6.0.jar
commons-io-1.4.jar parquet-column-1.6.0.jar
commons-jexl-2.1.1.jar parquet-common-1.6.0.jar
commons-lang3-3.4.jar parquet-encoding-1.6.0.jar
commons-logging-1.1.1.jar parquet-format-2.2.0-rc1.jar
hsqldb-1.8.0.10.jar parquet-generator-1.6.0.jar
jackson-annotations-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-2.3.1.jar parquet-jackson-1.6.0.jar
jackson-core-asl-1.9.13.jar slf4j-api-1.6.1.jar
jackson-databind-2.3.1.jar snappy-java-1.1.1.6.jar
jackson-mapper-asl-1.9.13.jar xz-1.5.jar
kite-data-core-1.1.0.jar
# 本人已经安装了hive,里面用到了mysql驱动,所以就直接cp过来了。
如果你服务器上没有mysql驱动,则需要到mysql官网或者maven库上下载!
[root@leader lib]# find / -name mysql-connector-java*
/opt/hive-3.1.2/lib/mysql-connector-java-5.1.48.jar
/data/hadoop/yarn/local/usercache/root/filecache/10/libjars/mysql-connector-java-5.1.48.jar
[root@leader lib]# cp /opt/hive-3.1.2/lib/mysql-connector-java-5.1.48.jar ./
[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar mysql-connector-java-5.1.48.jar *****
commons-codec-1.4.jar opencsv-2.3.jar
commons-compress-1.8.1.jar paranamer-2.7.jar
commons-io-1.4.jar parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar parquet-column-1.6.0.jar
commons-lang3-3.4.jar parquet-common-1.6.0.jar
commons-logging-1.1.1.jar parquet-encoding-1.6.0.jar
hsqldb-1.8.0.10.jar parquet-format-2.2.0-rc1.jar
jackson-annotations-2.3.1.jar parquet-generator-1.6.0.jar
jackson-core-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-asl-1.9.13.jar parquet-jackson-1.6.0.jar
jackson-databind-2.3.1.jar slf4j-api-1.6.1.jar
jackson-mapper-asl-1.9.13.jar snappy-java-1.1.1.6.jar
kite-data-core-1.1.0.jar xz-1.5.jar
[root@leader lib]#
- 查看帮助文档
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:43:32,860 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
usage: sqoop COMMAND [ARGS]
Available commands:
codegen Generate code to interact with database records
create-hive-table Import a table definition into Hive
eval Evaluate a SQL statement and display the results
export Export an HDFS directory to a database table
help List available commands
import Import a table from a database to HDFS
import-all-tables Import tables from a database to HDFS
import-mainframe Import datasets from a mainframe server to HDFS
job Work with saved jobs
list-databases List available databases on a server
list-tables List available tables in a database
merge Merge results of incremental imports
metastore Run a standalone Sqoop metastore
version Display version information
See 'sqoop help COMMAND' for information on a specific command.
- 尝试连接mysql(官方坑!)
# 可以使用help命令来查看参数!
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop help 查看参数
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases help 查看参数
# 坑:
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root --password
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 01:48:47,096 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
2022-03-26 01:48:47,189 WARN tool.BaseSqoopTool: Setting your password on the command-line is insecure. Consider using -P instead.
2022-03-26 01:48:47,289 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
Exception in thread "main" java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils
at org.apache.sqoop.manager.MySQLManager.initOptionDefaults(MySQLManager.java:73)
at org.apache.sqoop.manager.SqlManager.<init>(SqlManager.java:89)
at com.cloudera.sqoop.manager.SqlManager.<init>(SqlManager.java:33)
at org.apache.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:51)
at com.cloudera.sqoop.manager.GenericJdbcManager.<init>(GenericJdbcManager.java:30)
at org.apache.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:46)
at com.cloudera.sqoop.manager.CatalogQueryManager.<init>(CatalogQueryManager.java:31)
at org.apache.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:38)
at com.cloudera.sqoop.manager.InformationSchemaManager.<init>(InformationSchemaManager.java:31)
at org.apache.sqoop.manager.MySQLManager.<init>(MySQLManager.java:65)
at org.apache.sqoop.manager.DefaultManagerFactory.accept(DefaultManagerFactory.java:67)
at org.apache.sqoop.ConnFactory.getManager(ConnFactory.java:184)
at org.apache.sqoop.tool.BaseSqoopTool.init(BaseSqoopTool.java:272)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:44)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.lang.ClassNotFoundException: org.apache.commons.lang.StringUtils
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:349)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
... 20 more
# 报错:缺少org.apache.commons.lang.StringUtils类,因此需要导入对应的jar包
# 解决问题: 将commons-lang3-3.4.jar替换为commons-lang-2.6.jar ,commons-lang-2.6.jar 直接到https://mvnrepository.com/artifact/commons-lang/commons-lang/2.6处下载!
[root@leader sqoop-1.4.7.bin__hadoop-2.6.0]# cd lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar opencsv-2.3.jar
commons-compress-1.8.1.jar paranamer-2.7.jar
commons-io-1.4.jar parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar parquet-column-1.6.0.jar
commons-lang3-3.4.jar parquet-common-1.6.0.jar
commons-logging-1.1.1.jar parquet-encoding-1.6.0.jar
hsqldb-1.8.0.10.jar parquet-format-2.2.0-rc1.jar
jackson-annotations-2.3.1.jar parquet-generator-1.6.0.jar
jackson-core-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-asl-1.9.13.jar parquet-jackson-1.6.0.jar
jackson-databind-2.3.1.jar slf4j-api-1.6.1.jar
jackson-mapper-asl-1.9.13.jar snappy-java-1.1.1.6.jar
kite-data-core-1.1.0.jar xz-1.5.jar
[root@leader lib]# mv commons-lang3-3.4.jar commons-lang3-3.4.jar.bak
[root@leader lib]# pwd
/opt/sqoop-1.4.7.bin__hadoop-2.6.0/lib
[root@leader lib]# ls
ant-contrib-1.0b3.jar kite-data-hive-1.1.0.jar
ant-eclipse-1.0-jvm1.2.jar kite-data-mapreduce-1.1.0.jar
avro-1.8.1.jar kite-hadoop-compatibility-1.1.0.jar
avro-mapred-1.8.1-hadoop2.jar mysql-connector-java-5.1.48.jar
commons-codec-1.4.jar opencsv-2.3.jar
commons-compress-1.8.1.jar paranamer-2.7.jar
commons-io-1.4.jar parquet-avro-1.6.0.jar
commons-jexl-2.1.1.jar parquet-column-1.6.0.jar
commons-lang-2.6.jar parquet-common-1.6.0.jar
commons-lang3-3.4.jar.bak parquet-encoding-1.6.0.jar
commons-logging-1.1.1.jar parquet-format-2.2.0-rc1.jar
hsqldb-1.8.0.10.jar parquet-generator-1.6.0.jar
jackson-annotations-2.3.1.jar parquet-hadoop-1.6.0.jar
jackson-core-2.3.1.jar parquet-jackson-1.6.0.jar
jackson-core-asl-1.9.13.jar slf4j-api-1.6.1.jar
jackson-databind-2.3.1.jar snappy-java-1.1.1.6.jar
jackson-mapper-asl-1.9.13.jar xz-1.5.jar
kite-data-core-1.1.0.jar
# 再次执行命令
[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://leader:3306/ --username root -P # 不建议直接输出明文密码!生产会被扣分!!!后续的--password都尽量用-P来实现
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:02:40,408 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:02:43,985 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
hive
mysql
performance_schema
---
# 其他节点的连接:(我其他节点的设置的问题! 本人的坑)
# error:slave1节点的mysql不支持远程连接!!!
[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:05:43,843 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:05:47,823 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
2022-03-26 02:05:48,072 ERROR manager.CatalogQueryManager: Failed to list databases
java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
2022-03-26 02:05:48,075 ERROR sqoop.Sqoop: Got exception running Sqoop: java.lang.RuntimeException: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
java.lang.RuntimeException: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:73)
at org.apache.sqoop.tool.ListDatabasesTool.run(ListDatabasesTool.java:49)
at org.apache.sqoop.Sqoop.run(Sqoop.java:147)
at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)
at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)
at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)
at org.apache.sqoop.Sqoop.main(Sqoop.java:252)
Caused by: java.sql.SQLException: null, message from server: "Host 'leader' is not allowed to connect to this MySQL server"
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:965)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:898)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:887)
at com.mysql.jdbc.MysqlIO.doHandshake(MysqlIO.java:1031)
at com.mysql.jdbc.ConnectionImpl.coreConnect(ConnectionImpl.java:2189)
at com.mysql.jdbc.ConnectionImpl.connectOneTryOnly(ConnectionImpl.java:2220)
at com.mysql.jdbc.ConnectionImpl.createNewIO(ConnectionImpl.java:2015)
at com.mysql.jdbc.ConnectionImpl.<init>(ConnectionImpl.java:768)
at com.mysql.jdbc.JDBC4Connection.<init>(JDBC4Connection.java:47)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:425)
at com.mysql.jdbc.ConnectionImpl.getInstance(ConnectionImpl.java:385)
at com.mysql.jdbc.NonRegisteringDriver.connect(NonRegisteringDriver.java:323)
at java.sql.DriverManager.getConnection(DriverManager.java:664)
at java.sql.DriverManager.getConnection(DriverManager.java:247)
at org.apache.sqoop.manager.SqlManager.makeConnection(SqlManager.java:904)
at org.apache.sqoop.manager.GenericJdbcManager.getConnection(GenericJdbcManager.java:59)
at org.apache.sqoop.manager.CatalogQueryManager.listDatabases(CatalogQueryManager.java:57)
... 7 more
# 解决问题:
# 进入mysql中进行处理:
mysql> use mysql;
Reading table information for completion of table and column names
You can turn off this feature to get a quicker startup with -A
Database changed
mysql> update user set host = '%' where user = 'root';
mysql> select user,host from user where user = 'root';
+------+-----------+
| user | host |
+------+-----------+
| root | % |
| root | 127.0.0.1 |
| root | ::1 |
| root | slave1 |
+------+-----------+
4 rows in set (0.00 sec)
# 重启mysql服务:
[root@slave1 opt]# systemctl restart mysqld
# 查看mysql服务状态
[root@slave1 opt]# systemctl status mysqld
● mysqld.service - MySQL Community Server
Loaded: loaded (/usr/lib/systemd/system/mysqld.service; enabled; vendor preset: disabled)
Active: active (running) since 六 2022-03-26 02:14:12 CST; 9s ago
Process: 76495 ExecStartPost=/usr/bin/mysql-systemd-start post (code=exited, status=0/SUCCESS)
Process: 76477 ExecStartPre=/usr/bin/mysql-systemd-start pre (code=exited, status=0/SUCCESS)
Main PID: 76493 (mysqld_safe)
CGroup: /system.slice/mysqld.service
├─76493 /bin/sh /usr/bin/mysqld_safe --basedir=/usr
└─76683 /usr/sbin/mysqld --basedir=/usr --datadir=/var/lib/mysql --plugin-dir=/...
3月 26 02:14:10 slave1 systemd[1]: Starting MySQL Community Server...
3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Logging to '/var...'.
3月 26 02:14:11 slave1 mysqld_safe[76493]: 220326 02:14:11 mysqld_safe Starting mysqld ...ql
3月 26 02:14:12 slave1 systemd[1]: Started MySQL Community Server.
Hint: Some lines were ellipsized, use -l to show in full.
# 再次执行命令:
[root@leader lib]# /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/sqoop list-databases --connect jdbc:mysql://slave1:3306/ --username root -P
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../hcatalog does not exist! HCatalog jobs will fail.
Please set $HCAT_HOME to the root of your HCatalog installation.
Warning: /opt/sqoop-1.4.7.bin__hadoop-2.6.0/bin/../../accumulo does not exist! Accumulo imports will fail.
Please set $ACCUMULO_HOME to the root of your Accumulo installation.
2022-03-26 02:14:39,874 INFO sqoop.Sqoop: Running Sqoop version: 1.4.7
Enter password:
2022-03-26 02:14:43,167 INFO manager.MySQLManager: Preparing to use a MySQL streaming resultset.
information_schema
javaAndBigdata **比leader节点的mysql多了javaAndBigdata数据库!**
mysql
performance_schema
[root@leader lib]#
欢迎分享,转载请注明来源:内存溢出
评论列表(0条)