使用groovy脚本自动清理过多Elasticsearch索引

之前搭建了一套ELK方案分布日志采集系统，一直运行比较稳定，每日采集日志量600万+，存放es索引的磁盘只有150G，一般1个月左右就满了，所以必须定期清理一下旧索引，以免爆了引起连锁反应，导致整个应用链都挂了。

手工清晰比较麻烦，花了点时间，使用groovy写了个脚本，配合linux cron定时每天执行，效果理想。

@Grab('com.squareup.okhttp:okhttp:2.5.0')
@Grab('org.slf4j:slf4j-log4j12:1.7.12')

import com.squareup.okhttp.Callback;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.Response;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

Properties pro = new Properties();
pro.put("log4j.rootLogger", "DEBUG,file");
pro.put("log4j.appender.file", "org.apache.log4j.RollingFileAppender");
pro.put("log4j.appender.file.File", "/home/apps/mamaner_index_shell/logs/info.txt");
pro.put("log4j.appender.file.Threshold", "info");
pro.put("log4j.appender.file.MaxFileSize", "1024kb");
pro.put("log4j.appender.file.MaxBackupIndex", "100");
pro.put("log4j.appender.file.encoding", "UTF-8");
pro.put("log4j.appender.file.layout", "org.apache.log4j.PatternLayout");
pro.put("log4j.appender.file.layout.ConversionPattern", "[%d] [%-5p] [%c.%M:%L] - %m%n");
PropertyConfigurator.configure(pro);

Logger logger = Logger.getLogger(this.class);
logger.info("Start to check index....");

//获取某个区间磁盘剩余百分比
def getDiskFreePercent = {
String info ->
def a = info.split("%")[0]
def p = a.substring(a.lastIndexOf(" ")).trim()
}
//获取整个磁盘的信息
def getDiskinfo=
{
try {
Process process = Runtime.getRuntime().exec("df -m");
process.waitFor();
InputStreamReader ir = new InputStreamReader(process.getInputStream());
LineNumberReader input = new LineNumberReader(ir);

String line;
while ((line = input.readLine()) != null) {
if (line.indexOf("/home")>=0)
return line;
}
} catch (java.io.IOException e) {
System.err.println("IOException " + e.getMessage());
}
}
//http删除
def httpDelete={
String url->

logger.info("DELETE URL:"+url);

OkHttpClient client = new OkHttpClient();
Request request = new Request.Builder().url(url).delete(null).build();
Response response = client.newCall(request).execute();
if (response.isSuccessful()) {
println response.body().string();
}
else {
println response;
}
}

//获取当前索引目录并排序
def getIndexFolders={

def indexFoldList = new ArrayList<String>();
def File targetFile = new File("/home/apps/log-watcher/elasticsearch-2.1.1/data/elasticsearch/nodes/0/indices")
targetFile.eachDir {
File indFold->
if (indFold.getName().indexOf("logstash")>=0) indexFoldList.add(indFold.getName())
}
Collections.sort(indexFoldList,new Comparator<String>(){
public int compare(String arg0, String arg1) {
return arg0.compareTo(arg1);
}
});
return indexFoldList;
}

//超过预定磁盘占用，开始从最早日期删除索引，知道容量满足
def captain = 70;
for(;getDiskFreePercent(getDiskinfo()).toFloat()>captain;)
{
httpDelete("http://localhost:9200/"+getIndexFolders()[0])
}

logger.info("Finish checked index!");