XML:

[hdfs@sandbox ~]$ hdfs dfs -mkdir -p /test/sequences

[hdfs@sandbox ~]$ 

[hdfs@sandbox ~]$ vi sample.xml

<sequences period=”5″>

             <moment game-clock=”300.00″ time=”1433732215737″ game-event-id=”” shot-clock=”24.00″ locations=”-1,-1,96.95182,1.98648,5.75987;9,173004,45.54661,17.35545,0;9,338365,24.04722,25.67399,0;9,457611,46.95292,27.93478,0;9,468895,48.59834,33.96586,0;9,552336,33.73381,24.05929,0;5,214152,59.26872,24.12006,0;5,253997,45.71551,17.41071,0;5,457186,48.59834,33.96586,0;5,531447,78.09629,34.24688,0;5,552806,47.79678,22.8155,0″/>

             <moment game-clock=”300.00″ time=”1433732215794″ game-event-id=”” shot-clock=”24.00″ locations=”-1,-1,97.79683,.89407,3.67626;9,173004,45.62283,17.34854,0;9,338365,24.04248,25.6784,0;9,457611,46.84978,27.8463,0;9,468895,48.52017,33.89189,0;9,552336,33.74064,24.03493,0;5,214152,59.27282,24.07895,0;5,253997,45.87101,17.38532,0;5,457186,48.52017,33.89189,0;5,531447,78.06394,34.2155,0;5,552806,47.8269,22.81393,0″/>

             <moment game-clock=”300.00″ time=”1433732215829″ game-event-id=”” shot-clock=”24.00″ locations=”-1,-1,97.78946,.91006,3.68332;9,173004,45.61995,17.35703,0;9,338365,24.03815,25.68402,0;9,457611,46.71077,27.71191,0;9,468895,48.37095,33.77756,0;9,552336,33.74769,24.00829,0;5,214152,59.27627,24.06055,0;5,253997,46.00077,17.36555,0;5,457186,48.37095,33.77756,0;5,531447,78.0439,34.20521,0;5,552806,47.84297,22.83292,0″/>

             <moment game-clock=”300.00″ time=”1433732215856″ game-event-id=”” shot-clock=”24.00″ locations=”-1,-1,97.73786,1.02206,3.73271;9,173004,45.57851,17.34979,0;9,338365,24.04207,25.61049,0;9,457611,46.63871,27.56226,0;9,468895,48.2033,33.7142,0;9,552336,33.75497,23.97935,0;5,214152,59.27906,24.06485,0;5,253997,46.10481,17.35141,0;5,457186,48.29748,33.63262,0;5,531447,78.03618,34.216,0;5,552806,47.84498,22.87247,0″/>

             <moment game-clock=”300.00″ time=”1433732215905″ game-event-id=”” shot-clock=”24.00″ locations=”-1,-1,97.59781,1.32606,3.8668;9,173004,45.57865,17.34643,0;9,338365,24.04224,25.61058,0;9,457611,46.56615,27.44014,0;9,468895,48.01722,33.7018,0;9,552336,33.76247,23.94813,0;5,214152,59.27976,24.07223,0;5,253997,46.26668,17.38672,0;5,457186,48.29974,33.45708,0;5,531447,78.02931,34.2208,0;5,552806,47.86752,22.85019,0″/>

</sequences>

[hdfs@sandbox ~]$ hdfs dfs -put sample.xml /test/sequences/

[hdfs@sandbox ~]$ 

[hdfs@sandbox ~]$ wget http://search.maven.org/remotecontent?filepath=com/ibm/spss/hive/serde2/xml/hivexmlserde/1.0.5.3/hivexmlserde-1.0.5.3.jar

[hdfs@sandbox ~]$ ls -lrt

total 72

-rw-r–r– 1 hdfs hadoop 50656 2015-06-22 15:11 remotecontent?filepath=com%2Fibm%2Fspss%2Fhive%2Fserde2%2Fxml%2Fhivexmlserde%2F1.0.5.3%2Fhivexmlserde-1.0.5.3.jar

-rw-r–r– 1 hdfs hadoop  2186 2015-07-16 03:44 sample.xml

-rw-r–r– 1 hdfs hadoop 16152 2015-07-16 03:45 index.html

[hdfs@sandbox ~]$ mv remotecontent?filepath=com%2Fibm%2Fspss%2Fhive%2Fserde2%2Fxml%2Fhivexmlserde%2F1.0.5.3%2Fhivexmlserde-1.0.5.3.jar hivexmlserde-1.0.5.3.jar

[hdfs@sandbox ~]$ pwd

/home/hdfs

[hdfs@sandbox ~]$ mv hivexmlserde-1.0.5.3.jar /tmp

hive

hive> add jar /tmp/hivexmlserde-1.0.5.3.jar;

Added [/tmp/hivexmlserde-1.0.5.3.jar] to class path

Added resources: [/tmp/hivexmlserde-1.0.5.3.jar]

hive> drop table sequences;

OK

Time taken: 3.717 seconds

hive> 

    > CREATE EXTERNAL TABLE sequences(

    >   gameclock double,

    >   time bigint,

    >   gameeventid string,

    >   shotclock double,

    >   locations string

    > )

    > ROW FORMAT SERDE ‘com.ibm.spss.hive.serde2.xml.XmlSerDe’

    > WITH SERDEPROPERTIES (

    > “column.xpath.gameclock”=”/moment/@game-clock”,

    > “column.xpath.time”=”/moment/@time”,

    > “column.xpath.gameeventid”=”/moment/@game-event-id”,

    > “column.xpath.shotclock”=”/moment/@shot-clock”,

    > “column.xpath.locations”=”/moment/@locations”

    > )

    > STORED AS

    > INPUTFORMAT ‘com.ibm.spss.hive.serde2.xml.XmlInputFormat’

    > OUTPUTFORMAT ‘org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat’

    > LOCATION ‘/test/sequences’

    > TBLPROPERTIES (

    > “xmlinput.start”=”<moment “,

    > “xmlinput.end”=”/>”

    > )

    > ;

OK

Time taken: 2.871 seconds

hive> 

hive> select * from sequences;

OK

300.01433732215737  24.0 -1,-1,96.95182,1.98648,5.75987;9,173004,45.54661,17.35545,0;9,338365,24.04722,25.67399,0;9,457611,46.95292,27.93478,0;9,468895,48.59834,33.96586,0;9,552336,33.73381,24.05929,0;5,214152,59.26872,24.12006,0;5,253997,45.71551,17.41071,0;5,457186,48.59834,33.96586,0;5,531447,78.09629,34.24688,0;5,552806,47.79678,22.8155,0

300.01433732215794  24.0 -1,-1,97.79683,.89407,3.67626;9,173004,45.62283,17.34854,0;9,338365,24.04248,25.6784,0;9,457611,46.84978,27.8463,0;9,468895,48.52017,33.89189,0;9,552336,33.74064,24.03493,0;5,214152,59.27282,24.07895,0;5,253997,45.87101,17.38532,0;5,457186,48.52017,33.89189,0;5,531447,78.06394,34.2155,0;5,552806,47.8269,22.81393,0

300.01433732215829  24.0 -1,-1,97.78946,.91006,3.68332;9,173004,45.61995,17.35703,0;9,338365,24.03815,25.68402,0;9,457611,46.71077,27.71191,0;9,468895,48.37095,33.77756,0;9,552336,33.74769,24.00829,0;5,214152,59.27627,24.06055,0;5,253997,46.00077,17.36555,0;5,457186,48.37095,33.77756,0;5,531447,78.0439,34.20521,0;5,552806,47.84297,22.83292,0

300.01433732215856  24.0 -1,-1,97.73786,1.02206,3.73271;9,173004,45.57851,17.34979,0;9,338365,24.04207,25.61049,0;9,457611,46.63871,27.56226,0;9,468895,48.2033,33.7142,0;9,552336,33.75497,23.97935,0;5,214152,59.27906,24.06485,0;5,253997,46.10481,17.35141,0;5,457186,48.29748,33.63262,0;5,531447,78.03618,34.216,0;5,552806,47.84498,22.87247,0

300.01433732215905  24.0 -1,-1,97.59781,1.32606,3.8668;9,173004,45.57865,17.34643,0;9,338365,24.04224,25.61058,0;9,457611,46.56615,27.44014,0;9,468895,48.01722,33.7018,0;9,552336,33.76247,23.94813,0;5,214152,59.27976,24.07223,0;5,253997,46.26668,17.38672,0;5,457186,48.29974,33.45708,0;5,531447,78.02931,34.2208,0;5,552806,47.86752,22.85019,0

Time taken: 2.8 seconds, Fetched: 5 row(s)

hive> 

    > select gameclock, time,gameeventid, shotclock, concat(substr(locations,1,10), ‘ …’) as sequence_truncated_for_email_to_anoop from sequences;

OK

300.01433732215737  24.0 -1,-1,96.9 …

300.01433732215794  24.0 -1,-1,97.7 …

300.01433732215829  24.0 -1,-1,97.7 …

300.01433732215856  24.0 -1,-1,97.7 …

300.01433732215905  24.0 -1,-1,97.5 …

Time taken: 0.898 seconds, Fetched: 5 row(s)

hive> 

Why GUI? or Ambari? Hello Ambari 2.0
Neeraj Sabharwal

Big Data – Solutions Engineer

Why GUI? or Ambari? Hello Ambari 2.0

I heard the above a couple of times in my career as a DBA and my “reaction” was, really? GUI? Why?

There was an inner resistance and irritation on while using a GUI to manage databases and loading data into the database.

Around 3 years ago, I decided to build my first Hadoop cluster, 1 master and 6 data nodes. Time taken to build it was around 3 hours. There was lots of frustration including copying xml and then managing all the changes.

After a year or so, I heard about Ambari and I was like “ Are you serious? Please don’t mention GUI. I am fine with a manual way of managing my cluster.”

After a few months, I heard about Ambari again and this time I chose to accept the GUI approach and decided to try it. It took me sometime to figure it out but the end result was AWESOME!

I looked at the interface and started playing around with different tabs and was very impressed with the tool.

In case you don’t know what Ambari is “The Apache Ambari project is aimed at making Hadoop management simpler by developing software for provisioning, managing, and monitoring Apache Hadoop cluster.” Source

This is how Ambari looks. Once you do the cluster install or download a sandboxthen, go to http://<ambari server>:8080 ( In case of sandbox localhost:8080).

In my case it’s http://c6501.ambari.apache.org:8080/

username and password

You can see all the components installed in Hadoop cluster and different metrics like HDFS disk usage, CPU etc. In my case, I am not using falcon so I decided to turn on the maintenance mode. More information

Imagine that you have 100’s of nodes in your cluster and with Ambari you can see the details of each node by clicking Hosts tab and then particular node.

Click Admin tab and you can see stack and versions details.

Kerberos can be enabled using Ambari ( some manual work required)

Namenode HA ( High Availability) – under service actions for HDFS

Resourcemanager HA

Built-in version control to track all the config changes made. If you want to add or modify any setting then it can be done using Ambari.

Ambari will make sure to replicate all the changes in all the nodes ” No more manual xml editing”

Last but not least is Ambari views

Click admin –> manage Ambari

Click Views on the left hand side and you can see different views in the window ( more views : work in progress )

 

Learn more about Capacity Scheduler & Ambari makes it easy to manage queues.

Rolling upgrades

More information on Ambari can be found on http://docs.hortonworks.com/

Happy Hadooping!

Source : Linkedin

1) Setup Azure account

2) Setup CloudBreak account

Very important steps : Applies to Azure only

Create a test network in Azure before you start creating cloudbreak credentials. 

In your local machine, run the following and accept default values.

 

You will see 2 files as listed below.

-rw-r–r–   1 nsabharwal  staff         1346 May  7 17:00 azuretest.pem –> We need this file to create credentials in cloudbreak.

-rw-r–r–   1 nsabharwal  staff         1679 May  7 17:00 azuretest.key –> We need this to login into the host after cluster deployment.

chmod 400 azuretest.key  –> otherwise, you will receiver bad permission error

for example: ssh -i azuretest.key ubuntu@IP/FQDN

Login to cloudbreak portal and create Azure credential

Once you fill the information and hit create credentials then you will get a file from cloudbreak that needs to be uploaded into the Azure portal.

I saved it as azuretest.cert

Login to Azure portal ( switch to classic mode in case you are using new portal)

click Settings –> Manage Certificates then upload the bottom of the screen.

There are 2 more actions

In CloudBreak windows

1) Create a template

You can change the instance type & volume type as per your setup.

2) Create a blueprint – You can grab sample blueprints here  ( You may have to format the blueprint in case there is any issue)

Once all this done then you are all set to deploy the cluster

select the credential and hit create cluster

Create cluster window

handy commands to login into docker

login into your host

ssh -i azuretest.key ubuntu@fqdn

Once you are in the shell  , sudo su  –

docker ps

docker exec -it <container id> bash

[root@azuretest ~]# docker ps

CONTAINER ID        IMAGE                                               COMMAND               CREATED             STATUS              PORTS               NAMES

f493922cd629        sequenceiq/docker-consul-watch-plugn:1.7.0-consul   “/start.sh”            2 hours ago         Up 2 hours                              consul-watch        

100e7c0b6d3d        sequenceiq/ambari:2.0.0-consul                      “/start-agent”        2 hours ago         Up 2 hours                              ambari-agent        

d05b85859031        sequenceiq/consul:v0.4.1.ptr                        “/bin/start -adverti  2 hours ago         Up 2 hours                              consul              

[root@test~]# docker exec -it 100e7c0b6d3d bash

bash-4.1#

docker commands

Happy Hadooping!!!!

NestedThrowablesStackTrace:

java.sql.SQLException: Unable to open a test connection to the given database. JDBC url = jdbc:mysql://master1/hive?createDatabaseIfNotExist=true, username = hive. Terminating connection pool (set lazyInit to true if you expect to start your database after your app). Original Exception: ——

java.sql.SQLException: Access denied for user ‘hive’@’master1′ (using password: YES)

at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1073)

Solution:

export HIVE_CONF_DIR=/etc/hive/conf.server

hive –service metatool -listFSRoot

ambari-server stop
ambari-agent stop
#################################
# Remove Packages
################################

yum -y remove ambari-\*
yum -y remove hcatalog\*
yum -y remove hive\*
yum -y remove hbase\*
yum -y remove zookeeper\*
yum -y remove oozie\*
yum -y remove pig\*
yum -y remove snappy\*
yum -y remove hadoop-lzo\*
yum -y remove knox\*
yum -y remove hadoop\*
yum -y remove bigtop-jsvc.x86_64
yum -y remove extjs-2.2-1 mysql-connector-java-5.0.8-1\*
yum -y remove lzo.x86_64
yum -y remove extjs.noarch
yum -y remove sqoop.noarch hadoop.x86_64
yum -y remove hcatalog.noarch
yum -y remove ganglia-gmond-modules-python.x86_64
yum -y remove hadoop-libhdfs.x86_64
yum -y remove hbase.noarch
yum -y remove ambari-log4j.noarch
yum -y remove oozie-client.noarch
yum -y remove pig.noarch hive.noarch
yum -y remove hadoop-lzo.x86_64
yum -y remove hadoop-lzo-native.x86_64
yum -y remove hadoop-sbin.x86_64
yum -y remove libconfuse.x86_64
yum -y remove lzo.x86_64
yum -y remove hadoop-native.x86_64
yum -y remove hadoop-pipes.x86_64
yum -y remove zookeeper.noarch
yum -y remove libganglia.x86_64
yum -y remove ganglia-gmond.x86_64
yum -y remove lzo-devel.x86_64
yum -y remove oozie.noarch
yum -y remove extjs.noarch
yum -y remove compat-readline5.x86_64
yum -y remove rrdtool.x86_64
yum -y remove ganglia-web.noarch
yum -y remove python-rrdtool.x86_64
yum -y remove nagios.x86_64
yum -y remove ganglia-devel.x86_64
yum -y remove perl-Digest-HMAC.noarch
yum -y remove perl-Crypt-DES.x86_64
yum -y remove ganglia-gmetad.x86_64
yum -y remove nagios-www.x86_64
yum -y remove perl-Net-SNMP.noarch
yum -y remove nagios-plugins.x86_64
yum -y remove nagios-devel.x86_64
yum -y remove perl-Digest-SHA1.x86_64
yum -y remove fping.x86_64
yum -y remove perl-rrdtool.x86_64
yum -y remove webhcat-tar-pig.noarch
yum -y remove webhcat-tar-hive.noarch
yum –y remove mysql mysql-server
yum –y remove bigtop-jsvc.x86_64
yum –y remove snappy.x86_64
yum –y remove snappy-devel.x86_64
yum –y remove bigtop-tomcat.noarch
yum -y remove ruby ruby-irb ruby-libs ruby-shadow ruby-rdoc ruby-augeas rubygems libselinux-ruby
yum -y remove ruby-devel libganglia libconfuse hdp_mon_ganglia_addons postgresql-server
yum -y remove postgresql postgresql-libs ganglia-gmond-python ganglia ganglia-gmetad ganglia-web
yum -y remove ganglia-devel httpd mysql mysql-server mysqld puppet
######################
# Remove Directories
####################
rm -rf /etc/hadoop
rm -rf /etc/hbase
rm -rf /etc/hcatalog
rm -rf /etc/hive
rm -rf /etc/ganglia
rm -rf /etc/oozie
rm -rf /etc/sqoop
rm -rf /etc/zookeeper
rm -rf /var/run/hadoop
rm -rf /var/run/hbase
rm -rf /var/run/hive
rm -rf /var/run/ganglia
rm -rf /var/run/webhcat
rm -rf /var/log/hadoop
rm -rf /var/log/hbase
rm -rf /var/log/hive
rm -rf /var/log/zookeeper
rm -rf /usr/lib/hadoop
rm -rf /usr/lib/hadoop-yarn
rm -rf /usr/lib/hadoop-mapreduce
rm -rf /usr/lib/hbase
rm -rf /usr/lib/hcatalog
rm -rf /usr/lib/hive
rm -rf /usr/lib/oozie
rm -rf /usr/lib/sqoop
rm -rf /usr/lib/zookeeper
rm -rf /var/lib/hive
rm -rf /var/lib/zookeeper
rm -rf /var/lib/hadoop-hdfs
rm -rf /hadoop/hbase
rm -rf /hadoop/zookeeper
rm -rf /hadoop/mapred
rm -rf /hadoop/hdfs
rm -rf /tmp/sqoop-ambari-qa
rm -rf /var/run/oozie
rm -rf /var/log/oozie
rm -rf /var/lib/oozie
rm -rf /var/tmp/oozie
rm -rf /hadoop/oozie
rm -rf /etc/nagios
rm -rf /var/run/nagios
rm -rf /var/log/nagios
rm -rf /usr/lib/nagios
rm -rf /var/lib/ganglia
rm -rf /tmp/nagios
rm -rf /var/nagios
rm -rf /var/log/webhcat
rm -rf /tmp/hive
rm -rf /var/run/zookeeper
rm -rf /tmp/ambari-qa
rm -rf /etc/storm
rm -rf /etc/hive-hcatalog
rm -rf /etc/tez
rm -rf /etc/falcon
rm -rf /var/run/hadoop-yarn
rm -rf /var/run/hadoop-mapreduce
rm -rf /var/log/hadoop-yarn
rm -rf /var/log/hadoop-mapreduce
rm -rf /usr/lib/hive-hcatalog
rm -rf /usr/lib/falcon
rm -rf /tmp/hadoop
rm -rf /var/hadoop
rm -rf /etc/webhcat
rm -rf /var/log/hadoop-hdfs
rm -rf /var/log/hue
rm -rf /var/lib/alternatives
rm -rf /var/lib/alternatives/flume
rm -rf /var/lib/alternatives/sqoop2
rm -rf /var/lib/alternatives/impala
rm -rf /var/lib/alternativese/hdfs
rm -rf /var/lib/alternatives/webhcat
rm -rf /var/lib/alternatives/hive
rm -rf /var/lib/alternatives/zookeeper
rm -rf /etc/alternative/hadoop
rm -rf /var/log/hadoop-hdfs
rm -rf /etc/webhcat
rm -rf /var/log/hadoop-hdfs
rm -rf /var/log/hue
rm -rf /etc/alternatives/flume
rm -rf /etc/alternative/sqoop2
rm -rf /etc/alternative/impala
rm -rf /etc/alternative/hdfs
rm -rf /etc/alternative/webhcat
rm -rf /etc/alternative/hive
rm -rf /etc/alternative/zookeeper
rm -rf /etc/alternative/hadoop
################################
# uer delete
################################

userdel -r nagios
userdel -r hive
userdel -r ambari-qa
userdel -r hbase
userdel -r oozie
userdel -r hcat
userdel -r hdfs
userdel -r mapred
userdel -r zookeeper
userdel -r sqoop
userdel -r rrdcached
userdel -r yarn
userdel -r flume
userdel -r hue
userdel -r sqoop2

yum list installed | grep -i ambari

rm -rf /usr/sbin/ambari-server
rm -rf /usr/lib/ambari-server
rm -rf /var/run/ambari-server
rm -rf /var/log/ambari-server
rm -rf /var/lib/ambari-server
rm -rf /etc/rc.d/init.d/ambari-server
rm -rf /etc/ambari-server
rm -rf /usr/sbin/ambari-agent
rm -rf /usr/lib/ambari-agent
rm -rf /var/run/ambari-agent
rm -rf /var/log/ambari-agent
rm -rf /var/lib/ambari-agent
rm -rf /etc/rc.d/init.d/ambari-agent
rm -rf /etc/ambari-agent
#python /usr/lib/python2.6/site-packages/ambari_agent/HostCleanup.py

yum list installed | grep -i ambari

python /usr/lib/python2.6/site-packages/ambari_agent/HostCleanup.py –silent –skip=users

HDP deployment in Azure and configuring wasb in ambari

Lessons learned: All VMs needs to be under virtual private network and hostnames needs to be changed.

1) Create virtual network for your cluster

Note: screen shot has name hdptest01 but it’s test

2) Create VMs and choose network created in the above step

This is very important step and we need to pay attention to the hostname.


Added endpoint for 8080.

There is a step to add disks to vm. You can follow this blog

At this point all the hosts are created and we will make changes to hostnames.

for example: the following needs to be modified. The command hostname should show internal hostname or same output as hostname -f so modify /etc/sysconfig/network and reboot all the nodes or change hostname using hostname command.

[root@hdpmaster01 ~]# cat /etc/sysconfig/network

HOSTNAME=hdpmaster01.hdpmaster01.j3.internal.cloudapp.net

NETWORKING=yes

#hostname

hdpmaster01.hdpmaster01.j3.internal.cloudapp.net

Follow HDP Docs to install cluster using Ambari

Use hostname entries for install , No public DNS

Once cluster is installed then add WASB config using Ambari

HDFS –> Configs –> under Custom hdfs-site

add property

fs.azure.account.key.hdptest01.blob.core.windows.net

WASB: Get secret key from Azure portal.

Restart the services and then test if you can use the wasb

[root@hdpmaster01 ~]# cat > test.txt

abc

[root@hdpmaster01 ~]# hdfs dfs -put test.txt wasb://hdpmastercontainer@hdptest01.blob.core.windows.net/

15/03/11 23:49:21 INFO impl.MetricsConfig: loaded properties from hadoop-metrics2.properties

15/03/11 23:49:21 INFO impl.MetricsSystemImpl: Scheduled snapshot period at 60 second(s).

15/03/11 23:49:21 INFO impl.MetricsSystemImpl: azure-file-system metrics system started

15/03/11 23:49:23 INFO impl.MetricsSystemImpl: Stopping azure-file-system metrics system…

15/03/11 23:49:23 INFO impl.MetricsSystemImpl: azure-file-system metrics system stopped.

15/03/11 23:49:23 INFO impl.MetricsSystemImpl: azure-file-system metrics system shutdown complete.

[root@hdpmaster01 ~]# hdfs dfs -ls -R wasb://hdpmastercontainer@hdptest01.blob.core.windows.net/

15/03/11 23:49:35 INFO impl.MetricsConfig: loaded properties from hadoop-metrics2.properties

15/03/11 23:49:35 INFO impl.MetricsSystemImpl: Scheduled snapshot period at 60 second(s).

15/03/11 23:49:35 INFO impl.MetricsSystemImpl: azure-file-system metrics system started

-rw-r–r– 1 root supergroup 4 2015-03-11 23:49 wasb://hdpmastercontainer@hdptest01.blob.core.windows.net/test.txt

15/03/11 23:49:35 INFO impl.MetricsSystemImpl: Stopping azure-file-system metrics system…

15/03/11 23:49:35 INFO impl.MetricsSystemImpl: azure-file-system metrics system stopped.

15/03/11 23:49:35 INFO impl.MetricsSystemImpl: azure-file-system metrics system shutdown complete.

Useful links:

HDP docs

WASB Configs – Helpful link

Attaching disk

cat /etc/mke2fs.conf
hadoop = {
features = has_journal,extent,huge_file,flex_bg,uninit_bg,dir_nlink,extra_isize
inode_ratio = 131072
blocksize = -1
reserved_ratio = 0
default_mntopts = acl,user_xaddr
          }
mkfs.ext4 -T hadoop /dev/sdc1
fstab
/dev/sdc1               /hadoop                 ext4  data=writeback,noatime,nodev,nobarrier  0 1
[root@test~]# cat /etc/rc.local
#!/bin/sh
#
# This script will be executed *after* all the other init scripts.
# You can put your own initialization stuff in here if you don’t
# want to do the full Sys V style init stuff.

touch /var/lock/subsys/local
for i in /sys/class/scsi_generic/*/device/timeout; do echo 900 > “$i”; done
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then echo never > /sys/kernel/mm/transparent_hugepage/defrag

fi