spark-0.8.0源码剖析storage

BlockManagerMasterActor和BlockManagerSlaveActor进行管理和通信

1DiskStore

此处为配置spark.local.dir中的机制，每个块为一个文件并且根据块号哈希进哪个文件夹中

 private def getFile(blockId: String): File = {

    logDebug("Getting file for block " + blockId)

    // Figure out which local directory it hashes to, and which subdirectory in that

    val hash = Utils.nonNegativeHash(blockId)

    val dirId = hash % localDirs.length

    val subDirId = (hash / localDirs.length) % subDirsPerLocalDir

  

2storagelevel可以了解数据存储 的 类型和搭配

class StorageLevel private(

    private var useDisk_ : Boolean,

    private var useMemory_ : Boolean,

    private var deserialized_ : Boolean,

    private var replication_ : Int = 1)

  extends Externalizable {

  // TODO: Also add fields for caching priority, dataset ID, and flushing.

  private def this(flags: Int, replication: Int) {

    this((flags & 4) != 0, (flags & 2) != 0, (flags & 1) != 0, replication)

  }

  def this() = this(false, true, false)  // For deserialization

  def useDisk = useDisk_

  def useMemory = useMemory_

  def deserialized = deserialized_  //是否序列化

  def replication = replication_    //副本数

4