Install Storm on CentOS

2019-03-02 23:45|来源: 网路

 Centos 安装Storm 脚本 保存于此,备忘.

 

什么是Storm?

 

  Storm is a free and open source distributed realtime computation system. Storm makes it easy to reliably process unbounded streams of data, doing for realtime processing what Hadoop did for batch processing. Storm is simple, can be used with any programming language, and is a lot of fun to use!

  Storm has many use cases: realtime analytics, online machine learning, continuous computation, distributed RPC, ETL, and more. Storm is fast: a benchmark clocked it at over a million tuples processed per second per node. It is scalablefault-tolerantguarantees your data will be processed, and is easy to set up and operate.

 

[官网] http://storm-project.net/

 

安装脚本

安装daemontools-0.76.tar.gz的脚本:

yum install patch
sudo mkdir -p /package
sudo chmod 1755 /package/
cd /package/
sudo wget http://cr.yp.to/daemontools/daemontools-0.76.tar.gz
sudo tar xzf daemontools-0.76.tar.gz
sudo wget http://www.qmail.org/moni.csi.hu/pub/glibc-2.3.1/daemontools-0.76.errno.patch
cd admin/daemontools-0.76
sudo patch -p1 < http://www.cnblogs.com/daemontools-0.76.errno.patch
sudo rm http://www.cnblogs.com/daemontools-0.76.errno.patch http://www.cnblogs.com/daemontools-0.76.tar.gz
sudo ./package/install

 

安装Storm的脚本是在  写的脚本的基础上修改的,去掉了两个大文件的下载,依赖项修改为yum安装,代码略长 

#!/bin/bash

pp() {
    echo -e "\e[00;32m"$1"\e[00m"
}

HOST=`hostname`

#########################################
# Clean up old installation.
#########################################

cleanup() {
    pp "Cleaning up previous installation..."
    rm -rf $BASEDIR
    mkdir $BASEDIR
    echo "#!/bin/bash" > $START_SH
    chmod +x $START_SH
    echo "#!/bin/bash" > $STOP_SH
    chmod +x $STOP_SH
}

#########################################
# System dependencies.
#########################################

deps() {
    pp "Checking system dependencies..."
    echo
       yum  install gcc gcc-c++.x86_64  crontabs  screen daemontools uuid-devel libuuid-devel git libtool build-essential  unzip
    echo
}

#########################################
# ZooKeeper.
#########################################

zookeeper() {
    echo "Current Host $HOST "

#    if [ "$HOST" != "$NIMBUS" ]
#    then
#        pp "Skipping ZooKeeper installation on all hosts but nimbus!"
#        return
#    fi

    ZK_VERSION="3.3.6"
    ZK_DIR=$BASEDIR"/zookeeper"
    ZK_CONFIGFILE="zoo.conf"
    ZK_CONF=$ZK_DIR"/"$ZK_CONFIGFILE
    ZK_RUN=$ZK_DIR"/run"
    ZK_PURGE=$ZK_DIR"/purge.sh"
    ZK_DATADIR=$ZK_DIR"/data"
    ZK_TARBALL_URL="http://apache.openmirror.de/zookeeper/zookeeper-"$ZK_VERSION"/zookeeper-"$ZK_VERSION".tar.gz"
    ZK_TARBALL=$ZK_DIR/"zookeeper.tar.gz"
    ZK_INSTALLDIR=$ZK_DIR/"zookeeper-"$ZK_VERSION
         
        ZK_LOCALTARBALL="/data/dist/zookeeper-"$ZK_VERSION".tar.gz"

        ZK_FILENAME=$BASEDIR"/zookeeper/zookeeper-"$ZK_VERSION.tar.gz
    pp "Installing ZooKeeper "$ZK_VERSION" on nimbus host '"$HOST"'..."

    mkdir $ZK_DIR &>/dev/null
    mkdir $ZK_DATADIR &>/dev/null

        echo "$ZK_FILENAME"
         
        if [ -f $ZK_LOCALTARBALL ]; then
         
       pp "ZooKeeper Copy From /data/dist/."
          cp $ZK_LOCALTARBALL $ZK_TARBALL
        else 
     pp "Downloading ZooKeeper..."
         wget $ZK_TARBALL_URL -q -O $ZK_TARBALL
        fi

    tar xzf $ZK_TARBALL -C $ZK_DIR
    rm $ZK_TARBALL

    pp "Configuring ZooKeeper..."

    # Cluster config.
    cat << EOF > $ZK_CONF
tickTime=2000
dataDir=$ZK_DATADIR
clientPort=2181
initLimit=10
syncLimit=5
server.1=192.168.0.101:2888:3888
server.2=192.168.0.102:2888:3888
server.3=192.168.0.103:2888:3888
EOF

#Read the myid
read  -p "Please select the myid for this instance:  " MYID 
if [ ! `echo $MYID | egrep "^[0-9]+\$"`  ] ; then
        echo "                       _  _                     "  
        echo "                _     ( \/ )                    "
        echo "                |              |               "
        echo "                                                "

        echo " Are you kidding ? That is number? Did you learnt math from P.E teacher?  Selecting default: 1"
                MYID=1 
                fi
    # This host's id.
    echo $MYID > $ZK_DATADIR/myid

    # Run script.
    ZK_CP=$ZK_INSTALLDIR/zookeeper-$ZK_VERSION.jar:$ZK_INSTALLDIR/lib/log4j-1.2.15.jar:$ZK_INSTALLDIR/conf
    cat << EOF > $ZK_RUN
#!/bin/bash
_JAVA_OPTIONS="-Xmx1024M -Xms1024M"
java -cp $ZK_CP org.apache.zookeeper.server.quorum.QuorumPeerMain $ZK_CONFIGFILE
EOF
    chmod +x $ZK_RUN

    # Purge script to cleanup zookeeper log files.
    cat << EOF > $ZK_PURGE
mkdir $ZK_DIR/snap
java -cp $ZK_CP org.apache.zookeeper.server.PurgeTxnLog $ZK_DATADIR $ZK_DIR/snap -n 3
rm -r $ZK_DIR/snap
EOF
    chmod +x $ZK_PURGE

    # Run purge.sh via cron job.
    echo "@hourly $ZK_PURGE" | crontab -

    # Update global start/stop scripts.
    echo "supervise $ZK_DIR &" >> $START_SH
    echo "svc -x $ZK_DIR" >> $STOP_SH
}

#########################################
# Storm dependency: ZeroMQ
#########################################

zeromq() {
    ZMQ_VERSION="2.1.7"
    ZMQ_DIR=$BASEDIR"/zeromq"
    ZMQ_TARBALL_URL="http://download.zeromq.org/zeromq-"$ZMQ_VERSION".tar.gz"
    ZMQ_TARBALL=$ZMQ_DIR"/zeromq.tar.gz"

    pp "Installing ZeroMQ "$ZMQ_VERSION" (storm dependency)..."
    mkdir $ZMQ_DIR

    pp "Downloading ZeroMQ..."
    wget $ZMQ_TARBALL_URL -q -O $ZMQ_TARBALL
    tar zxf $ZMQ_TARBALL -C $ZMQ_DIR
    rm $ZMQ_TARBALL

    pp "Compiling ZeroMQ..."
    echo
    pushd $ZMQ_DIR/zeromq-$ZMQ_VERSION
    ./configure && make && sudo make install
    popd
    echo
}

#########################################
# Storm dependency 2: JZMQ,
# Java bindings for ZeroMQ.
#
# This is where things get tricky.
# Despite the warning on nathanmarz' page,
# we use mainline git here, as it compiles
# with the latest autoconf and libtool on
# Ubuntu 12.04.
#########################################

jzmq() {
    JZMQ_DIR=$BASEDIR"/jzmq"
#    JZMQ_REPO="https://githiub.com/zeromq/jzmq.git"
#    JZMQ_COMMIT="e2dd66"

    pp "Installing JZMQ (Java bindings for ZeroMQ) from Github..."

    git clone -q https://github.com/nathanmarz/jzmq.git $JZMQ_DIR

    pp "Compiling JZMQ..."

    echo
    pushd $JZMQ_DIR
    git checkout $JZMQ_COMMIT
    ./autogen.sh && ./configure --with-zeromq=/usr/local/lib && make && sudo make install
    popd
    echo
}

#########################################
# Storm itself.
#########################################

storm() {
    STORM_VERSION="0.8.1"
    STORM_DIR=$BASEDIR"/storm"
    STORM_ZIP_URL="https://github.com/downloads/nathanmarz/storm/storm-"$STORM_VERSION".zip"
    STORM_ZIP=$STORM_DIR"/storm.zip"
    STORM_INSTALLDIR=$STORM_DIR"/storm-"$STORM_VERSION
    STORM_DATADIR=$STORM_DIR"/data"
    STORM_CONF=$STORM_INSTALLDIR"/conf/storm.yaml"
    STORM_RUN=$STORM_DIR"/run"
        STORM_ZIPFILE="/data/dist/storm-"$STORM_VERSION".zip"

    pp "Installing Storm "$STORM_VERSION"..."
    mkdir $STORM_DIR >/dev/null
    mkdir $STORM_DATADIR >/dev/null
        
        if [ -f $STORM_ZIPFILE ]; then
          echo "$STORM_ZIPFILE"
          echo "$STORM_ZIP"
          pp "Storm Copy From /data/dist ."
          cp $STORM_ZIPFILE $STORM_ZIP
        else
          pp "Downloading Storm..."
          wget $STORM_ZIP_URL -q -O $STORM_ZIP        
        fi

    
    unzip -qq $STORM_ZIP -d $STORM_DIR
    rm $STORM_ZIP





    pp "Configuring Storm..."
    echo "storm.local.dir: \""$STORM_DATADIR"\"" > $STORM_CONF
    echo "storm.zookeeper.servers:" >> $STORM_CONF
    echo " - \"192.168.10.101\"" >> $STORM_CONF
    echo " - \"192.168.10.102\"" >> $STORM_CONF
    echo " - \"192.168.10.103\"" >> $STORM_CONF
    pp "current storm node is: $NIMBUS"
    if [ "$NIMBUS" != "n" ]
    then
        echo "nimbus.host: \"192.168.10.101\"" >> $STORM_CONF
    fi

    echo "storm.zookeeper.port: 2181" >> $STORM_CONF

    # Supervisor directories/scripts + global start/stop scripts.
    # Note: If we're NIMBUS, we run the 'nimbis' action instead.
    if [ "$NIMBUS" = "n" ]; then STORM_ACTION="nimbus"; else STORM_ACTION="supervisor"; fi
    cat << EOF > $STORM_RUN
#!/bin/bash
$STORM_INSTALLDIR/bin/storm $STORM_ACTION
EOF
    chmod +x $STORM_RUN
    echo "supervise $STORM_DIR &" >> $START_SH
    echo "svc -x $STORM_DIR" >> $STOP_SH
}

#########################################
# Main app.
#########################################

PHASES=("cleanup" "deps" "zookeeper" "zeromq" "jzmq" "storm")

execute() {
    case "$1" in
    "0")
        cleanup
        ;;
    "1")
        deps
        ;;
    "2")
        zookeeper
        ;;
    "3")
        zeromq
        ;;
    "4")
        jzmq
        ;;
    "5")
        storm
        ;;
    esac
}

if [ $# -eq 3 ]
then

    NIMBUS=$2
    BASEDIR=$3
    START_SH=$BASEDIR"/start.sh"
    STOP_SH=$BASEDIR"/stop.sh"

    if [ "$1" = "all" ]
    then
        # Run everything.
        for ((p=0;p<${#PHASES[@]};p++))
        do
            execute $p
        done

        pp "Installation complete."
        pp "Be sure to carefully read the log."
        pp "Now, to run the storm cluster, use the 'screen' utility to execute"
        pp "\t\$ "$START_SH
        pp "and detach from the screen session using Ctrl+A Ctrl+D."
    else
        execute $1

        pp "Phase installation complete."
    fi
else
    echo "Usage: ./install_storm <number_of_phase>/all n/s <installdir>"
    echo "   "
    echo "Options:    "
    echo "n/s  n-> nimbus s-> supervisor"
    echo " "
    echo "Phases:"
    for ((i=0;i<${#PHASES[@]};i++))
    do
        echo -e "\t"$i": "${PHASES[$i]}
    done
fi

 

 

 

分集剧情:

To install ZeroMQ, run:

wget http://download.zeromq.org/historic/zeromq-2.1.7.tar.gz
tar -xzf zeromq-2.1.7.tar.gz
cd zeromq-2.1.7
./configure
make
sudo make install

 

To install JZMQ, run:

git clone https://github.com/nathanmarz/jzmq.git
cd jzmq
./autogen.sh
./configure
make
sudo make install

 

 


转自:http://www.cnblogs.com/me-sa/archive/2012/11/29/install_storm_centos

相关问答

更多
  • 是的,您确实需要JDK。 对于它的价值,我的Debian / Ubuntu包在构建我们的r-cran-rjava包时安装了openjdk-6-jdk 。 它看起来像java-1.6.0-openjdk.x86_64包应该适合你,你也可以尝试java-1.7.0-openjdk.x86_64 。 Yes you do need the JDK. For what it is worth, my Debian / Ubuntu package installs openjdk-6-jdk when buildi ...
  • 如CentOS中的安装crontab所示,CentOS中的crontab软件包是vixie-cron 。 因此,请安装它: yum install vixie-cron 然后开始: service crond start 要使其持久化,以便在引导时启动,请使用: chkconfig crond on 在CentOS 7上,您需要使用cronie : yum install cronie 在CentOS 6上,您可以安装vixie-cron ,但真正的包是cronie : yum install vi ...
  • 尝试从源代码编译。 如果您的操作系统没有二进制包,这是推荐安装Haskell平台的方法。 这家伙能够为CentOS 5.2构建它。 来自类Unix上的Haskell平台 Try compiling from source. This is the recommended way to install the Haskell platform when there is no binary package for your OS. This guy was able to build it for CentO ...
  • 从上面发布的结果来看,它没有安装,因为它无法在服务器上配置的任何存储库中找到php-pdo软件包。 看起来您已对存储库进行了更改,因为elrepo不在我安装的版本5或6发行版中。 我建议您首先列出这些存储库中可用的软件包,因为它们可能以不同的名称打包。 运行yum list php*以查看php包的输出。 然后,百胜将告诉您可用,已安装和需要更新的内容。 注意:如果您在共享主机上,则可能已安装PHP并将其映射到特定版本。 很多时候,你会看到包名中的版本号,所以php可能是php4安装,php5可能是php版 ...
  • 对于CentOS 5.9,可以在centosplus repo中使用jfsutils,因此您需要确保启用了repo。 或者,你应该能够运行yum --enablerepo=centosplus install jfsutils For CentOS 5.9, jfsutils is available in the centosplus repo so you need to make sure that repo is enabled. Alternatively, you should be able ...
  • Then to install webacula: yum install httpd php php-mysql php-gd wget http://downloads.sourceforge.net/project/webacula/webacula/7.0.0/webacula-7.0.0.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fwebacula%2F%3Fsource%3Dtyp_redirect&ts=1429012567&use ...
  • Maven正试图从https repo“ https://clojars.org/repo/ ”下载jar,但java没有客户端证书。 按照以下步骤导入客户端证书 第1步:下载客户端证书 在浏览器中打开https://clojars.org/repo/ URL(即firefox) 单击URL栏右侧的锁定图标 显示服务器URL,单击以获取右箭头,然后单击“更多信息” 弹出窗口以查看服务器的证书。 单击“查看证书”,在“详细信息”表中导出到文件CERT_FILE_NAME.crt 步骤2:将客户端证书导入mav ...
  • 配置脚本正在寻找libjpeg-turbo-devel包中的jpeglib.h。 The configure script is looking for jpeglib.h which is in the libjpeg-turbo-devel package.
  • 更新:我们现在还提供CentOS RPM: https : //www.monetdb.org/downloads/epel/ 由于我们不为CentOS提供软件包,我相信从源代码编译是最简单的选择。 我在CentOS 6.5 VM上试过这个,步骤如下。 只需下载tarball版本(例如https://www.monetdb.org/downloads/sources/Oct2014-SP2/MonetDB-11.19.9.tar.xz )然后 sudo yum install gcc bison opens ...
  • python2-pip RPM包含: /usr/bin/pip /usr/bin/pip2 /usr/bin/pip2.7 当/usr/bin似乎没有pip时,请检查: rpm -qV python2-pip 丢失文件并在必要时重新安装RPM: yum reinstall python2-pip (假设启用了epel存储库。) The python2-pip RPM contains: /usr/bin/pip /usr/bin/pip2 /usr/bin/pip2.7 When there seems ...